From bb987cd64b118044a4f3351c377516813514ee19 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 20 Apr 2022 16:06:46 +0100 Subject: [PATCH 01/50] Lazy loading fixes (#2362) * Return some more usefully wrapped errors when doing sync * Remove unnecessary error check * Couple of guards around `sql.ErrNoRows` * Nolint --- syncapi/streams/stream_pdu.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index ddc2f55c2..0f11d55f6 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -3,6 +3,7 @@ package streams import ( "context" "database/sql" + "fmt" "sync" "time" @@ -205,6 +206,7 @@ func (p *PDUStreamProvider) IncrementalSync( return newPos } +// nolint:gocyclo func (p *PDUStreamProvider) addRoomDeltaToResponse( ctx context.Context, device *userapi.Device, @@ -228,13 +230,16 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( eventFilter, true, true, ) if err != nil { - return r.From, err + if err == sql.ErrNoRows { + return r.To, nil + } + return r.From, fmt.Errorf("p.DB.RecentEvents: %w", err) } recentEvents := p.DB.StreamEventsToEvents(device, recentStreamEvents) delta.StateEvents = removeDuplicates(delta.StateEvents, recentEvents) // roll back prevBatch, err := p.DB.GetBackwardTopologyPos(ctx, recentStreamEvents) if err != nil { - return r.From, err + return r.From, fmt.Errorf("p.DB.GetBackwardTopologyPos: %w", err) } // If we didn't return any events at all then don't bother doing anything else. @@ -268,15 +273,12 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( } if stateFilter.LazyLoadMembers { - if err != nil { - return r.From, err - } delta.StateEvents, err = p.lazyLoadMembers( ctx, delta.RoomID, true, limited, stateFilter.IncludeRedundantMembers, device, recentEvents, delta.StateEvents, ) - if err != nil { - return r.From, err + if err != nil && err != sql.ErrNoRows { + return r.From, fmt.Errorf("p.lazyLoadMembers: %w", err) } } From 54e7ea41c688a0bcc89150c99045b2dcdf8d0b12 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 20 Apr 2022 16:51:37 +0100 Subject: [PATCH 02/50] Eliminate more SQL no row errors in sync API (#2363) * Handle `sql.ErrNoRows` in main `/sync` codepaths * Catch more --- syncapi/storage/shared/syncserver.go | 34 ++++++++++++++++++++++++++-- syncapi/streams/stream_pdu.go | 7 ++++-- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index 91eba44e1..2143fd672 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -688,6 +688,9 @@ func (d *Database) GetStateDeltas( // user has ever interacted with — joined to, kicked/banned from, left. memberships, err := d.CurrentRoomState.SelectRoomIDsWithAnyMembership(ctx, txn, userID) if err != nil { + if err == sql.ErrNoRows { + return nil, nil, nil + } return nil, nil, err } @@ -705,17 +708,23 @@ func (d *Database) GetStateDeltas( // get all the state events ever (i.e. for all available rooms) between these two positions stateNeeded, eventMap, err := d.OutputEvents.SelectStateInRange(ctx, txn, r, stateFilter, allRoomIDs) if err != nil { + if err == sql.ErrNoRows { + return nil, nil, nil + } return nil, nil, err } state, err := d.fetchStateEvents(ctx, txn, stateNeeded, eventMap) if err != nil { + if err == sql.ErrNoRows { + return nil, nil, nil + } return nil, nil, err } // find out which rooms this user is peeking, if any. // We do this before joins so any peeks get overwritten peeks, err := d.Peeks.SelectPeeksInRange(ctx, txn, userID, device.ID, r) - if err != nil { + if err != nil && err != sql.ErrNoRows { return nil, nil, err } @@ -726,6 +735,9 @@ func (d *Database) GetStateDeltas( var s []types.StreamEvent s, err = d.currentStateStreamEventsForRoom(ctx, txn, peek.RoomID, stateFilter) if err != nil { + if err == sql.ErrNoRows { + continue + } return nil, nil, err } state[peek.RoomID] = s @@ -753,6 +765,9 @@ func (d *Database) GetStateDeltas( var s []types.StreamEvent s, err = d.currentStateStreamEventsForRoom(ctx, txn, roomID, stateFilter) if err != nil { + if err == sql.ErrNoRows { + continue + } return nil, nil, err } state[roomID] = s @@ -803,6 +818,9 @@ func (d *Database) GetStateDeltasForFullStateSync( // user has ever interacted with — joined to, kicked/banned from, left. memberships, err := d.CurrentRoomState.SelectRoomIDsWithAnyMembership(ctx, txn, userID) if err != nil { + if err == sql.ErrNoRows { + return nil, nil, nil + } return nil, nil, err } @@ -819,7 +837,7 @@ func (d *Database) GetStateDeltasForFullStateSync( deltas := make(map[string]types.StateDelta) peeks, err := d.Peeks.SelectPeeksInRange(ctx, txn, userID, device.ID, r) - if err != nil { + if err != nil && err != sql.ErrNoRows { return nil, nil, err } @@ -828,6 +846,9 @@ func (d *Database) GetStateDeltasForFullStateSync( if !peek.Deleted { s, stateErr := d.currentStateStreamEventsForRoom(ctx, txn, peek.RoomID, stateFilter) if stateErr != nil { + if stateErr == sql.ErrNoRows { + continue + } return nil, nil, stateErr } deltas[peek.RoomID] = types.StateDelta{ @@ -841,10 +862,16 @@ func (d *Database) GetStateDeltasForFullStateSync( // Get all the state events ever between these two positions stateNeeded, eventMap, err := d.OutputEvents.SelectStateInRange(ctx, txn, r, stateFilter, allRoomIDs) if err != nil { + if err == sql.ErrNoRows { + return nil, nil, nil + } return nil, nil, err } state, err := d.fetchStateEvents(ctx, txn, stateNeeded, eventMap) if err != nil { + if err == sql.ErrNoRows { + return nil, nil, nil + } return nil, nil, err } @@ -869,6 +896,9 @@ func (d *Database) GetStateDeltasForFullStateSync( for _, joinedRoomID := range joinedRoomIDs { s, stateErr := d.currentStateStreamEventsForRoom(ctx, txn, joinedRoomID, stateFilter) if stateErr != nil { + if stateErr == sql.ErrNoRows { + continue + } return nil, nil, stateErr } deltas[joinedRoomID] = types.StateDelta{ diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index 0f11d55f6..df5fb8e08 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -341,12 +341,16 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( wantFullState bool, device *userapi.Device, ) (jr *types.JoinResponse, err error) { + jr = types.NewJoinResponse() // TODO: When filters are added, we may need to call this multiple times to get enough events. // See: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L316 recentStreamEvents, limited, err := p.DB.RecentEvents( ctx, roomID, r, eventFilter, true, true, ) if err != nil { + if err == sql.ErrNoRows { + return jr, nil + } return } @@ -430,12 +434,11 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( false, limited, stateFilter.IncludeRedundantMembers, device, recentEvents, stateEvents, ) - if err != nil { + if err != nil && err != sql.ErrNoRows { return nil, err } } - jr = types.NewJoinResponse() jr.Summary.JoinedMemberCount = &joinedCount jr.Summary.InvitedMemberCount = &invitedCount jr.Timeline.PrevBatch = prevBatch From 2258387d393426922344213948c6d814de53f465 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 20 Apr 2022 16:55:24 +0100 Subject: [PATCH 03/50] Update test list --- sytest-whitelist | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sytest-whitelist b/sytest-whitelist index aef18c512..979f12bf6 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -312,10 +312,10 @@ Inbound federation can return events Inbound federation can return missing events for world_readable visibility Inbound federation can return missing events for invite visibility Inbound federation can get public room list -POST /rooms/:room_id/redact/:event_id as power user redacts message -POST /rooms/:room_id/redact/:event_id as original message sender redacts message -POST /rooms/:room_id/redact/:event_id as random user does not redact message -POST /redact disallows redaction of event in different room +PUT /rooms/:room_id/redact/:event_id/:txn_id as power user redacts message +PUT /rooms/:room_id/redact/:event_id/:txn_id as original message sender redacts message +PUT /rooms/:room_id/redact/:event_id/:txn_id as random user does not redact message +PUT /redact disallows redaction of event in different room An event which redacts itself should be ignored A pair of events which redact each other should be ignored Redaction of a redaction redacts the redaction reason From 144c060fa7e413d938200896b345b85ab975bad8 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 21 Apr 2022 13:31:43 +0100 Subject: [PATCH 04/50] Remove libp2p demos (#2337) * Remove libp2p demos * Put it back --- cmd/dendrite-demo-libp2p/main.go | 230 ------------ cmd/dendrite-demo-libp2p/mdnslistener.go | 62 ---- cmd/dendrite-demo-libp2p/p2pdendrite.go | 126 ------- cmd/dendrite-demo-libp2p/publicrooms.go | 153 -------- cmd/dendritejs/jsServer.go | 101 ------ cmd/dendritejs/keyfetcher.go | 87 ----- cmd/dendritejs/main.go | 270 -------------- cmd/dendritejs/main_noop.go | 24 -- cmd/dendritejs/publicrooms.go | 155 -------- go.mod | 10 +- go.sum | 442 ----------------------- 11 files changed, 1 insertion(+), 1659 deletions(-) delete mode 100644 cmd/dendrite-demo-libp2p/main.go delete mode 100644 cmd/dendrite-demo-libp2p/mdnslistener.go delete mode 100644 cmd/dendrite-demo-libp2p/p2pdendrite.go delete mode 100644 cmd/dendrite-demo-libp2p/publicrooms.go delete mode 100644 cmd/dendritejs/jsServer.go delete mode 100644 cmd/dendritejs/keyfetcher.go delete mode 100644 cmd/dendritejs/main.go delete mode 100644 cmd/dendritejs/main_noop.go delete mode 100644 cmd/dendritejs/publicrooms.go diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go deleted file mode 100644 index 26c8eb85f..000000000 --- a/cmd/dendrite-demo-libp2p/main.go +++ /dev/null @@ -1,230 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "crypto/ed25519" - "flag" - "fmt" - "io/ioutil" - "net/http" - "os" - "time" - - "github.com/gorilla/mux" - gostream "github.com/libp2p/go-libp2p-gostream" - p2phttp "github.com/libp2p/go-libp2p-http" - p2pdisc "github.com/libp2p/go-libp2p/p2p/discovery" - "github.com/matrix-org/dendrite/appservice" - "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/embed" - "github.com/matrix-org/dendrite/federationapi" - "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/dendrite/keyserver" - "github.com/matrix-org/dendrite/roomserver" - "github.com/matrix-org/dendrite/setup" - "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/setup/mscs" - "github.com/matrix-org/dendrite/userapi" - "github.com/matrix-org/gomatrixserverlib" - - "github.com/sirupsen/logrus" - - _ "github.com/mattn/go-sqlite3" -) - -func createKeyDB( - base *P2PDendrite, - db *gomatrixserverlib.KeyRing, -) { - mdns := mDNSListener{ - host: base.LibP2P, - keydb: db, - } - serv, err := p2pdisc.NewMdnsService( - base.LibP2PContext, - base.LibP2P, - time.Second*10, - "_matrix-dendrite-p2p._tcp", - ) - if err != nil { - panic(err) - } - serv.RegisterNotifee(&mdns) -} - -func createFederationClient( - base *P2PDendrite, -) *gomatrixserverlib.FederationClient { - fmt.Println("Running in libp2p federation mode") - fmt.Println("Warning: Federation with non-libp2p homeservers will not work in this mode yet!") - tr := &http.Transport{} - tr.RegisterProtocol( - "matrix", - p2phttp.NewTransport(base.LibP2P, p2phttp.ProtocolOption("/matrix")), - ) - return gomatrixserverlib.NewFederationClient( - base.Base.Cfg.Global.ServerName, base.Base.Cfg.Global.KeyID, - base.Base.Cfg.Global.PrivateKey, - gomatrixserverlib.WithTransport(tr), - ) -} - -func createClient( - base *P2PDendrite, -) *gomatrixserverlib.Client { - tr := &http.Transport{} - tr.RegisterProtocol( - "matrix", - p2phttp.NewTransport(base.LibP2P, p2phttp.ProtocolOption("/matrix")), - ) - return gomatrixserverlib.NewClient( - gomatrixserverlib.WithTransport(tr), - ) -} - -func main() { - instanceName := flag.String("name", "dendrite-p2p", "the name of this P2P demo instance") - instancePort := flag.Int("port", 8080, "the port that the client API will listen on") - flag.Parse() - - filename := fmt.Sprintf("%s-private.key", *instanceName) - _, err := os.Stat(filename) - var privKey ed25519.PrivateKey - if os.IsNotExist(err) { - _, privKey, _ = ed25519.GenerateKey(nil) - if err = ioutil.WriteFile(filename, privKey, 0600); err != nil { - fmt.Printf("Couldn't write private key to file '%s': %s\n", filename, err) - } - } else { - privKey, err = ioutil.ReadFile(filename) - if err != nil { - fmt.Printf("Couldn't read private key from file '%s': %s\n", filename, err) - _, privKey, _ = ed25519.GenerateKey(nil) - } - } - - cfg := config.Dendrite{} - cfg.Defaults(true) - cfg.Global.ServerName = "p2p" - cfg.Global.PrivateKey = privKey - cfg.Global.KeyID = gomatrixserverlib.KeyID(fmt.Sprintf("ed25519:%s", *instanceName)) - cfg.FederationAPI.FederationMaxRetries = 6 - cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName)) - cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName)) - cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName)) - cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName)) - cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName)) - cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName)) - cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) - cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-e2ekey.db", *instanceName)) - cfg.MSCs.MSCs = []string{"msc2836"} - cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName)) - if err = cfg.Derive(); err != nil { - panic(err) - } - - base := NewP2PDendrite(&cfg, "Monolith") - defer base.Base.Close() // nolint: errcheck - - accountDB := base.Base.CreateAccountsDB() - federation := createFederationClient(base) - keyAPI := keyserver.NewInternalAPI(&base.Base, &base.Base.Cfg.KeyServer, federation) - - rsAPI := roomserver.NewInternalAPI( - &base.Base, - ) - - userAPI := userapi.NewInternalAPI(&base.Base, accountDB, &cfg.UserAPI, nil, keyAPI, rsAPI, base.Base.PushGatewayHTTPClient()) - keyAPI.SetUserAPI(userAPI) - - asAPI := appservice.NewInternalAPI(&base.Base, userAPI, rsAPI) - rsAPI.SetAppserviceAPI(asAPI) - fsAPI := federationapi.NewInternalAPI( - &base.Base, federation, rsAPI, base.Base.Caches, nil, true, - ) - keyRing := fsAPI.KeyRing() - rsAPI.SetFederationAPI(fsAPI, keyRing) - provider := newPublicRoomsProvider(base.LibP2PPubsub, rsAPI) - err = provider.Start() - if err != nil { - panic("failed to create new public rooms provider: " + err.Error()) - } - - createKeyDB( - base, keyRing, - ) - - monolith := setup.Monolith{ - Config: base.Base.Cfg, - AccountDB: accountDB, - Client: createClient(base), - FedClient: federation, - KeyRing: keyRing, - - AppserviceAPI: asAPI, - FederationAPI: fsAPI, - RoomserverAPI: rsAPI, - UserAPI: userAPI, - KeyAPI: keyAPI, - ExtPublicRoomsProvider: provider, - } - monolith.AddAllPublicRoutes( - base.Base.ProcessContext, - base.Base.PublicClientAPIMux, - base.Base.PublicFederationAPIMux, - base.Base.PublicKeyAPIMux, - base.Base.PublicWellKnownAPIMux, - base.Base.PublicMediaAPIMux, - base.Base.SynapseAdminMux, - ) - if err := mscs.Enable(&base.Base, &monolith); err != nil { - logrus.WithError(err).Fatalf("Failed to enable MSCs") - } - - httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() - httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.Base.InternalAPIMux) - httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.Base.PublicClientAPIMux) - httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.Base.PublicMediaAPIMux) - embed.Embed(httpRouter, *instancePort, "Yggdrasil Demo") - - libp2pRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() - libp2pRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.Base.PublicFederationAPIMux) - libp2pRouter.PathPrefix(httputil.PublicKeyPathPrefix).Handler(base.Base.PublicKeyAPIMux) - libp2pRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.Base.PublicMediaAPIMux) - - // Expose the matrix APIs directly rather than putting them under a /api path. - go func() { - httpBindAddr := fmt.Sprintf(":%d", *instancePort) - logrus.Info("Listening on ", httpBindAddr) - logrus.Fatal(http.ListenAndServe(httpBindAddr, httpRouter)) - }() - // Expose the matrix APIs also via libp2p - if base.LibP2P != nil { - go func() { - logrus.Info("Listening on libp2p host ID ", base.LibP2P.ID()) - listener, err := gostream.Listen(base.LibP2P, "/matrix") - if err != nil { - panic(err) - } - defer func() { - logrus.Fatal(listener.Close()) - }() - logrus.Fatal(http.Serve(listener, libp2pRouter)) - }() - } - - // We want to block forever to let the HTTP and HTTPS handler serve the APIs - base.Base.WaitForShutdown() -} diff --git a/cmd/dendrite-demo-libp2p/mdnslistener.go b/cmd/dendrite-demo-libp2p/mdnslistener.go deleted file mode 100644 index c6105e52c..000000000 --- a/cmd/dendrite-demo-libp2p/mdnslistener.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "fmt" - "math" - - "github.com/libp2p/go-libp2p-core/host" - "github.com/libp2p/go-libp2p-core/peer" - "github.com/matrix-org/gomatrixserverlib" -) - -type mDNSListener struct { - keydb *gomatrixserverlib.KeyRing - host host.Host -} - -func (n *mDNSListener) HandlePeerFound(p peer.AddrInfo) { - if err := n.host.Connect(context.Background(), p); err != nil { - fmt.Println("Error adding peer", p.ID.String(), "via mDNS:", err) - } - if pubkey, err := p.ID.ExtractPublicKey(); err == nil { - raw, _ := pubkey.Raw() - if err := n.keydb.KeyDatabase.StoreKeys( - context.Background(), - map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{ - { - ServerName: gomatrixserverlib.ServerName(p.ID.String()), - KeyID: "ed25519:p2pdemo", - }: { - VerifyKey: gomatrixserverlib.VerifyKey{ - Key: gomatrixserverlib.Base64Bytes(raw), - }, - ValidUntilTS: math.MaxUint64 >> 1, - ExpiredTS: gomatrixserverlib.PublicKeyNotExpired, - }, - }, - ); err != nil { - fmt.Println("Failed to store keys:", err) - } - } - fmt.Println("Discovered", len(n.host.Peerstore().Peers())-1, "other libp2p peer(s):") - for _, peer := range n.host.Peerstore().Peers() { - if peer != n.host.ID() { - fmt.Println("-", peer) - } - } -} diff --git a/cmd/dendrite-demo-libp2p/p2pdendrite.go b/cmd/dendrite-demo-libp2p/p2pdendrite.go deleted file mode 100644 index ba1868b27..000000000 --- a/cmd/dendrite-demo-libp2p/p2pdendrite.go +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "fmt" - - "errors" - - pstore "github.com/libp2p/go-libp2p-core/peerstore" - record "github.com/libp2p/go-libp2p-record" - - "github.com/libp2p/go-libp2p" - circuit "github.com/libp2p/go-libp2p-circuit" - crypto "github.com/libp2p/go-libp2p-core/crypto" - routing "github.com/libp2p/go-libp2p-core/routing" - - host "github.com/libp2p/go-libp2p-core/host" - dht "github.com/libp2p/go-libp2p-kad-dht" - pubsub "github.com/libp2p/go-libp2p-pubsub" - "github.com/matrix-org/gomatrixserverlib" - - "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" -) - -// P2PDendrite is a Peer-to-Peer variant of BaseDendrite. -type P2PDendrite struct { - Base base.BaseDendrite - - // Store our libp2p object so that we can make outgoing connections from it - // later - LibP2P host.Host - LibP2PContext context.Context - LibP2PCancel context.CancelFunc - LibP2PDHT *dht.IpfsDHT - LibP2PPubsub *pubsub.PubSub -} - -// NewP2PDendrite creates a new instance to be used by a component. -// The componentName is used for logging purposes, and should be a friendly name -// of the component running, e.g. SyncAPI. -func NewP2PDendrite(cfg *config.Dendrite, componentName string) *P2PDendrite { - baseDendrite := base.NewBaseDendrite(cfg, componentName) - - ctx, cancel := context.WithCancel(context.Background()) - - privKey, err := crypto.UnmarshalEd25519PrivateKey(cfg.Global.PrivateKey[:]) - if err != nil { - panic(err) - } - - //defaultIP6ListenAddr, _ := multiaddr.NewMultiaddr("/ip6/::/tcp/0") - var libp2pdht *dht.IpfsDHT - libp2p, err := libp2p.New(ctx, - libp2p.Identity(privKey), - libp2p.DefaultListenAddrs, - //libp2p.ListenAddrs(defaultIP6ListenAddr), - libp2p.DefaultTransports, - libp2p.Routing(func(h host.Host) (r routing.PeerRouting, err error) { - libp2pdht, err = dht.New(ctx, h) - if err != nil { - return nil, err - } - libp2pdht.Validator = libP2PValidator{} - r = libp2pdht - return - }), - libp2p.EnableAutoRelay(), - libp2p.EnableRelay(circuit.OptHop), - ) - if err != nil { - panic(err) - } - - libp2ppubsub, err := pubsub.NewFloodSub(context.Background(), libp2p, []pubsub.Option{ - pubsub.WithMessageSigning(true), - }...) - if err != nil { - panic(err) - } - - fmt.Println("Our public key:", privKey.GetPublic()) - fmt.Println("Our node ID:", libp2p.ID()) - fmt.Println("Our addresses:", libp2p.Addrs()) - - cfg.Global.ServerName = gomatrixserverlib.ServerName(libp2p.ID().String()) - - return &P2PDendrite{ - Base: *baseDendrite, - LibP2P: libp2p, - LibP2PContext: ctx, - LibP2PCancel: cancel, - LibP2PDHT: libp2pdht, - LibP2PPubsub: libp2ppubsub, - } -} - -type libP2PValidator struct { - KeyBook pstore.KeyBook -} - -func (v libP2PValidator) Validate(key string, value []byte) error { - ns, _, err := record.SplitKey(key) - if err != nil || ns != "matrix" { - return errors.New("not Matrix path") - } - return nil -} - -func (v libP2PValidator) Select(k string, vals [][]byte) (int, error) { - return 0, nil -} diff --git a/cmd/dendrite-demo-libp2p/publicrooms.go b/cmd/dendrite-demo-libp2p/publicrooms.go deleted file mode 100644 index 96e8ab5e1..000000000 --- a/cmd/dendrite-demo-libp2p/publicrooms.go +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "encoding/json" - "fmt" - "sync" - "sync/atomic" - "time" - - roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - - pubsub "github.com/libp2p/go-libp2p-pubsub" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" -) - -const MaintenanceInterval = time.Second * 10 - -type discoveredRoom struct { - time time.Time - room gomatrixserverlib.PublicRoom -} - -type publicRoomsProvider struct { - pubsub *pubsub.PubSub - topic *pubsub.Topic - subscription *pubsub.Subscription - foundRooms map[string]discoveredRoom // additional rooms we have learned about from the DHT - foundRoomsMutex sync.RWMutex // protects foundRooms - maintenanceTimer *time.Timer // - roomsAdvertised atomic.Value // stores int - rsAPI roomserverAPI.RoomserverInternalAPI -} - -func newPublicRoomsProvider(ps *pubsub.PubSub, rsAPI roomserverAPI.RoomserverInternalAPI) *publicRoomsProvider { - return &publicRoomsProvider{ - foundRooms: make(map[string]discoveredRoom), - pubsub: ps, - rsAPI: rsAPI, - } -} - -func (p *publicRoomsProvider) Start() error { - if topic, err := p.pubsub.Join("/matrix/publicRooms"); err != nil { - return err - } else if sub, err := topic.Subscribe(); err == nil { - p.topic = topic - p.subscription = sub - go p.MaintenanceTimer() - go p.FindRooms() - p.roomsAdvertised.Store(0) - } else { - return err - } - return nil -} - -func (p *publicRoomsProvider) MaintenanceTimer() { - if p.maintenanceTimer != nil && !p.maintenanceTimer.Stop() { - <-p.maintenanceTimer.C - } - p.Interval() -} - -func (p *publicRoomsProvider) Interval() { - p.foundRoomsMutex.Lock() - for k, v := range p.foundRooms { - if time.Since(v.time) > time.Minute { - delete(p.foundRooms, k) - } - } - p.foundRoomsMutex.Unlock() - if err := p.AdvertiseRooms(); err != nil { - fmt.Println("Failed to advertise room in DHT:", err) - } - p.foundRoomsMutex.RLock() - defer p.foundRoomsMutex.RUnlock() - fmt.Println("Found", len(p.foundRooms), "room(s), advertised", p.roomsAdvertised.Load(), "room(s)") - p.maintenanceTimer = time.AfterFunc(MaintenanceInterval, p.Interval) -} - -func (p *publicRoomsProvider) AdvertiseRooms() error { - ctx := context.Background() - var queryRes roomserverAPI.QueryPublishedRoomsResponse - // Query published rooms on our server. This will not invoke clientapi.ExtraPublicRoomsProvider - err := p.rsAPI.QueryPublishedRooms(ctx, &roomserverAPI.QueryPublishedRoomsRequest{}, &queryRes) - if err != nil { - util.GetLogger(ctx).WithError(err).Error("QueryPublishedRooms failed") - return err - } - ourRooms, err := roomserverAPI.PopulatePublicRooms(ctx, queryRes.RoomIDs, p.rsAPI) - if err != nil { - util.GetLogger(ctx).WithError(err).Error("PopulatePublicRooms failed") - return err - } - advertised := 0 - for _, room := range ourRooms { - if j, err := json.Marshal(room); err == nil { - if err := p.topic.Publish(context.TODO(), j); err != nil { - fmt.Println("Failed to publish public room:", err) - } else { - advertised++ - } - } - } - - p.roomsAdvertised.Store(advertised) - return nil -} - -func (p *publicRoomsProvider) FindRooms() { - for { - msg, err := p.subscription.Next(context.Background()) - if err != nil { - continue - } - received := discoveredRoom{ - time: time.Now(), - } - if err := json.Unmarshal(msg.Data, &received.room); err != nil { - fmt.Println("Unmarshal error:", err) - continue - } - fmt.Printf("received %+v \n", received) - p.foundRoomsMutex.Lock() - p.foundRooms[received.room.RoomID] = received - p.foundRoomsMutex.Unlock() - } -} - -func (p *publicRoomsProvider) Rooms() (rooms []gomatrixserverlib.PublicRoom) { - p.foundRoomsMutex.RLock() - defer p.foundRoomsMutex.RUnlock() - for _, dr := range p.foundRooms { - rooms = append(rooms, dr.room) - } - return -} diff --git a/cmd/dendritejs/jsServer.go b/cmd/dendritejs/jsServer.go deleted file mode 100644 index 4298c2ae9..000000000 --- a/cmd/dendritejs/jsServer.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build wasm -// +build wasm - -package main - -import ( - "bufio" - "fmt" - "net/http" - "net/http/httptest" - "strings" - "syscall/js" -) - -// JSServer exposes an HTTP-like server interface which allows JS to 'send' requests to it. -type JSServer struct { - // The router which will service requests - Mux http.Handler -} - -// OnRequestFromJS is the function that JS will invoke when there is a new request. -// The JS function signature is: -// function(reqString: string): Promise<{result: string, error: string}> -// Usage is like: -// const res = await global._go_js_server.fetch(reqString); -// if (res.error) { -// // handle error: this is a 'network' error, not a non-2xx error. -// } -// const rawHttpResponse = res.result; -func (h *JSServer) OnRequestFromJS(this js.Value, args []js.Value) interface{} { - // we HAVE to spawn a new goroutine and return immediately or else Go will deadlock - // if this request blocks at all e.g for /sync calls - httpStr := args[0].String() - promise := js.Global().Get("Promise").New(js.FuncOf(func(pthis js.Value, pargs []js.Value) interface{} { - // The initial callback code for new Promise() is also called on the critical path, which is why - // we need to put this in an immediately invoked goroutine. - go func() { - resolve := pargs[0] - resStr, err := h.handle(httpStr) - errStr := "" - if err != nil { - errStr = err.Error() - } - resolve.Invoke(map[string]interface{}{ - "result": resStr, - "error": errStr, - }) - }() - return nil - })) - return promise -} - -// handle invokes the http.ServeMux for this request and returns the raw HTTP response. -func (h *JSServer) handle(httpStr string) (resStr string, err error) { - req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(httpStr))) - if err != nil { - return - } - w := httptest.NewRecorder() - - h.Mux.ServeHTTP(w, req) - - res := w.Result() - var resBuffer strings.Builder - err = res.Write(&resBuffer) - return resBuffer.String(), err -} - -// ListenAndServe registers a variable in JS-land with the given namespace. This variable is -// a function which JS-land can call to 'send' HTTP requests. The function is attached to -// a global object called "_go_js_server". See OnRequestFromJS for more info. -func (h *JSServer) ListenAndServe(namespace string) { - globalName := "_go_js_server" - // register a hook in JS-land for it to invoke stuff - server := js.Global().Get(globalName) - if !server.Truthy() { - server = js.Global().Get("Object").New() - js.Global().Set(globalName, server) - } - - server.Set(namespace, js.FuncOf(h.OnRequestFromJS)) - - fmt.Printf("Listening for requests from JS on function %s.%s\n", globalName, namespace) - // Block forever to mimic http.ListenAndServe - select {} -} diff --git a/cmd/dendritejs/keyfetcher.go b/cmd/dendritejs/keyfetcher.go deleted file mode 100644 index cdf937649..000000000 --- a/cmd/dendritejs/keyfetcher.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build wasm -// +build wasm - -package main - -import ( - "context" - "fmt" - "time" - - "github.com/libp2p/go-libp2p-core/peer" - "github.com/matrix-org/gomatrixserverlib" -) - -const libp2pMatrixKeyID = "ed25519:libp2p-dendrite" - -type libp2pKeyFetcher struct { -} - -// FetchKeys looks up a batch of public keys. -// Takes a map from (server name, key ID) pairs to timestamp. -// The timestamp is when the keys need to be vaild up to. -// Returns a map from (server name, key ID) pairs to server key objects for -// that server name containing that key ID -// The result may have fewer (server name, key ID) pairs than were in the request. -// The result may have more (server name, key ID) pairs than were in the request. -// Returns an error if there was a problem fetching the keys. -func (f *libp2pKeyFetcher) FetchKeys( - ctx context.Context, - requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, -) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) { - res := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) - for req := range requests { - if req.KeyID != libp2pMatrixKeyID { - return nil, fmt.Errorf("FetchKeys: cannot fetch key with ID %s, should be %s", req.KeyID, libp2pMatrixKeyID) - } - - // The server name is a libp2p peer ID - peerIDStr := string(req.ServerName) - peerID, err := peer.Decode(peerIDStr) - if err != nil { - return nil, fmt.Errorf("Failed to decode peer ID from server name '%s': %w", peerIDStr, err) - } - pubKey, err := peerID.ExtractPublicKey() - if err != nil { - return nil, fmt.Errorf("Failed to extract public key from peer ID: %w", err) - } - pubKeyBytes, err := pubKey.Raw() - if err != nil { - return nil, fmt.Errorf("Failed to extract raw bytes from public key: %w", err) - } - b64Key := gomatrixserverlib.Base64Bytes(pubKeyBytes) - res[req] = gomatrixserverlib.PublicKeyLookupResult{ - VerifyKey: gomatrixserverlib.VerifyKey{ - Key: b64Key, - }, - ExpiredTS: gomatrixserverlib.PublicKeyNotExpired, - ValidUntilTS: gomatrixserverlib.AsTimestamp(time.Now().Add(24 * time.Hour * 365)), - } - } - return res, nil -} - -// FetcherName returns the name of this fetcher, which can then be used for -// logging errors etc. -func (f *libp2pKeyFetcher) FetcherName() string { - return "libp2pKeyFetcher" -} - -// no-op function for storing keys - we don't do any work to fetch them so don't bother storing. -func (f *libp2pKeyFetcher) StoreKeys(ctx context.Context, results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) error { - return nil -} diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go deleted file mode 100644 index 05e0f0ad9..000000000 --- a/cmd/dendritejs/main.go +++ /dev/null @@ -1,270 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build wasm -// +build wasm - -package main - -import ( - "crypto/ed25519" - "fmt" - "syscall/js" - - "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/appservice" - "github.com/matrix-org/dendrite/federationapi" - "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/dendrite/keyserver" - "github.com/matrix-org/dendrite/roomserver" - "github.com/matrix-org/dendrite/setup" - "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/userapi" - go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p" - - "github.com/matrix-org/gomatrixserverlib" - - "github.com/sirupsen/logrus" - - _ "github.com/matrix-org/go-sqlite3-js" -) - -var GitCommit string - -func init() { - fmt.Printf("[%s] dendrite.js starting...\n", GitCommit) -} - -const keyNameEd25519 = "_go_ed25519_key" - -func readKeyFromLocalStorage() (key ed25519.PrivateKey, err error) { - localforage := js.Global().Get("localforage") - if !localforage.Truthy() { - err = fmt.Errorf("readKeyFromLocalStorage: no localforage") - return - } - // https://localforage.github.io/localForage/ - item, ok := await(localforage.Call("getItem", keyNameEd25519)) - if !ok || !item.Truthy() { - err = fmt.Errorf("readKeyFromLocalStorage: no key in localforage") - return - } - fmt.Println("Found key in localforage") - // extract []byte and make an ed25519 key - seed := make([]byte, 32, 32) - js.CopyBytesToGo(seed, item) - - return ed25519.NewKeyFromSeed(seed), nil -} - -func writeKeyToLocalStorage(key ed25519.PrivateKey) error { - localforage := js.Global().Get("localforage") - if !localforage.Truthy() { - return fmt.Errorf("writeKeyToLocalStorage: no localforage") - } - - // make a Uint8Array from the key's seed - seed := key.Seed() - jsSeed := js.Global().Get("Uint8Array").New(len(seed)) - js.CopyBytesToJS(jsSeed, seed) - // write it - localforage.Call("setItem", keyNameEd25519, jsSeed) - return nil -} - -// taken from https://go-review.googlesource.com/c/go/+/150917 - -// await waits until the promise v has been resolved or rejected and returns the promise's result value. -// The boolean value ok is true if the promise has been resolved, false if it has been rejected. -// If v is not a promise, v itself is returned as the value and ok is true. -func await(v js.Value) (result js.Value, ok bool) { - if v.Type() != js.TypeObject || v.Get("then").Type() != js.TypeFunction { - return v, true - } - done := make(chan struct{}) - onResolve := js.FuncOf(func(this js.Value, args []js.Value) interface{} { - result = args[0] - ok = true - close(done) - return nil - }) - defer onResolve.Release() - onReject := js.FuncOf(func(this js.Value, args []js.Value) interface{} { - result = args[0] - ok = false - close(done) - return nil - }) - defer onReject.Release() - v.Call("then", onResolve, onReject) - <-done - return -} - -func generateKey() ed25519.PrivateKey { - // attempt to look for a seed in JS-land and if it exists use it. - priv, err := readKeyFromLocalStorage() - if err == nil { - fmt.Println("Read key from localStorage") - return priv - } - // generate a new key - fmt.Println(err, " : Generating new ed25519 key") - _, priv, err = ed25519.GenerateKey(nil) - if err != nil { - logrus.Fatalf("Failed to generate ed25519 key: %s", err) - } - if err := writeKeyToLocalStorage(priv); err != nil { - fmt.Println("failed to write key to localStorage: ", err) - // non-fatal, we'll just have amnesia for a while - } - return priv -} - -func createFederationClient(cfg *config.Dendrite, node *go_http_js_libp2p.P2pLocalNode) *gomatrixserverlib.FederationClient { - fmt.Println("Running in js-libp2p federation mode") - fmt.Println("Warning: Federation with non-libp2p homeservers will not work in this mode yet!") - tr := go_http_js_libp2p.NewP2pTransport(node) - - fed := gomatrixserverlib.NewFederationClient( - cfg.Global.ServerName, cfg.Global.KeyID, cfg.Global.PrivateKey, - gomatrixserverlib.WithTransport(tr), - ) - - return fed -} - -func createClient(node *go_http_js_libp2p.P2pLocalNode) *gomatrixserverlib.Client { - tr := go_http_js_libp2p.NewP2pTransport(node) - return gomatrixserverlib.NewClient( - gomatrixserverlib.WithTransport(tr), - ) -} - -func createP2PNode(privKey ed25519.PrivateKey) (serverName string, node *go_http_js_libp2p.P2pLocalNode) { - hosted := "/dns4/rendezvous.matrix.org/tcp/8443/wss/p2p-websocket-star/" - node = go_http_js_libp2p.NewP2pLocalNode("org.matrix.p2p.experiment", privKey.Seed(), []string{hosted}, "p2p") - serverName = node.Id - fmt.Println("p2p assigned ServerName: ", serverName) - return -} - -func main() { - cfg := &config.Dendrite{} - cfg.Defaults(true) - cfg.UserAPI.AccountDatabase.ConnectionString = "file:/idb/dendritejs_account.db" - cfg.AppServiceAPI.Database.ConnectionString = "file:/idb/dendritejs_appservice.db" - cfg.FederationAPI.Database.ConnectionString = "file:/idb/dendritejs_fedsender.db" - cfg.MediaAPI.Database.ConnectionString = "file:/idb/dendritejs_mediaapi.db" - cfg.RoomServer.Database.ConnectionString = "file:/idb/dendritejs_roomserver.db" - cfg.SyncAPI.Database.ConnectionString = "file:/idb/dendritejs_syncapi.db" - cfg.KeyServer.Database.ConnectionString = "file:/idb/dendritejs_e2ekey.db" - cfg.Global.JetStream.StoragePath = "file:/idb/dendritejs/" - cfg.Global.TrustedIDServers = []string{ - "matrix.org", "vector.im", - } - cfg.Global.KeyID = libp2pMatrixKeyID - cfg.Global.PrivateKey = generateKey() - - serverName, node := createP2PNode(cfg.Global.PrivateKey) - cfg.Global.ServerName = gomatrixserverlib.ServerName(serverName) - - if err := cfg.Derive(); err != nil { - logrus.Fatalf("Failed to derive values from config: %s", err) - } - base := setup.NewBaseDendrite(cfg, "Monolith") - defer base.Close() // nolint: errcheck - - accountDB := base.CreateAccountsDB() - federation := createFederationClient(cfg, node) - keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation) - userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, nil, keyAPI) - keyAPI.SetUserAPI(userAPI) - - fetcher := &libp2pKeyFetcher{} - keyRing := gomatrixserverlib.KeyRing{ - KeyFetchers: []gomatrixserverlib.KeyFetcher{ - fetcher, - }, - KeyDatabase: fetcher, - } - - rsAPI := roomserver.NewInternalAPI(base) - asQuery := appservice.NewInternalAPI( - base, userAPI, rsAPI, - ) - rsAPI.SetAppserviceAPI(asQuery) - fedSenderAPI := federationapi.NewInternalAPI(base, federation, rsAPI, base.Caches, keyRing, true) - rsAPI.SetFederationAPI(fedSenderAPI, keyRing) - p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node, fedSenderAPI, federation) - - psAPI := pushserver.NewInternalAPI(base) - - monolith := setup.Monolith{ - Config: base.Cfg, - AccountDB: accountDB, - Client: createClient(node), - FedClient: federation, - KeyRing: &keyRing, - - AppserviceAPI: asQuery, - FederationSenderAPI: fedSenderAPI, - RoomserverAPI: rsAPI, - UserAPI: userAPI, - KeyAPI: keyAPI, - PushserverAPI: psAPI, - //ServerKeyAPI: serverKeyAPI, - ExtPublicRoomsProvider: p2pPublicRoomProvider, - } - monolith.AddAllPublicRoutes( - base.ProcessContext, - base.PublicClientAPIMux, - base.PublicFederationAPIMux, - base.PublicKeyAPIMux, - base.PublicMediaAPIMux, - base.SynapseAdminMux, - ) - - httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() - httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux) - httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux) - httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux) - - libp2pRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() - libp2pRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux) - libp2pRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux) - - // Expose the matrix APIs via libp2p-js - for federation traffic - if node != nil { - go func() { - logrus.Info("Listening on libp2p-js host ID ", node.Id) - s := JSServer{ - Mux: libp2pRouter, - } - s.ListenAndServe("p2p") - }() - } - - // Expose the matrix APIs via fetch - for local traffic - go func() { - logrus.Info("Listening for service-worker fetch traffic") - s := JSServer{ - Mux: httpRouter, - } - s.ListenAndServe("fetch") - }() - - // We want to block forever to let the fetch and libp2p handler serve the APIs - select {} -} diff --git a/cmd/dendritejs/main_noop.go b/cmd/dendritejs/main_noop.go deleted file mode 100644 index 0cc7e47e5..000000000 --- a/cmd/dendritejs/main_noop.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build !wasm -// +build !wasm - -package main - -import "fmt" - -func main() { - fmt.Println("dendritejs: no-op when not compiling for WebAssembly") -} diff --git a/cmd/dendritejs/publicrooms.go b/cmd/dendritejs/publicrooms.go deleted file mode 100644 index 2e3339a45..000000000 --- a/cmd/dendritejs/publicrooms.go +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build wasm -// +build wasm - -package main - -import ( - "context" - "sync" - "time" - - "github.com/matrix-org/dendrite/federationapi/api" - go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" -) - -type libp2pPublicRoomsProvider struct { - node *go_http_js_libp2p.P2pLocalNode - providers []go_http_js_libp2p.PeerInfo - fedSender api.FederationInternalAPI - fedClient *gomatrixserverlib.FederationClient -} - -func NewLibP2PPublicRoomsProvider( - node *go_http_js_libp2p.P2pLocalNode, fedSender api.FederationInternalAPI, fedClient *gomatrixserverlib.FederationClient, -) *libp2pPublicRoomsProvider { - p := &libp2pPublicRoomsProvider{ - node: node, - fedSender: fedSender, - fedClient: fedClient, - } - node.RegisterFoundProviders(p.foundProviders) - return p -} - -func (p *libp2pPublicRoomsProvider) foundProviders(peerInfos []go_http_js_libp2p.PeerInfo) { - // work out the diff then poke for new ones - seen := make(map[string]bool, len(p.providers)) - for _, pr := range p.providers { - seen[pr.Id] = true - } - var newPeers []gomatrixserverlib.ServerName - for _, pi := range peerInfos { - if !seen[pi.Id] { - newPeers = append(newPeers, gomatrixserverlib.ServerName(pi.Id)) - } - } - if len(newPeers) > 0 { - var res api.PerformServersAliveResponse - // ignore errors, we don't care. - p.fedSender.PerformServersAlive(context.Background(), &api.PerformServersAliveRequest{ - Servers: newPeers, - }, &res) - } - - p.providers = peerInfos -} - -func (p *libp2pPublicRoomsProvider) Rooms() []gomatrixserverlib.PublicRoom { - return bulkFetchPublicRoomsFromServers(context.Background(), p.fedClient, p.homeservers()) -} - -func (p *libp2pPublicRoomsProvider) homeservers() []string { - result := make([]string, len(p.providers)) - for i := range p.providers { - result[i] = p.providers[i].Id - } - return result -} - -// bulkFetchPublicRoomsFromServers fetches public rooms from the list of homeservers. -// Returns a list of public rooms. -func bulkFetchPublicRoomsFromServers( - ctx context.Context, fedClient *gomatrixserverlib.FederationClient, homeservers []string, -) (publicRooms []gomatrixserverlib.PublicRoom) { - limit := 200 - // follow pipeline semantics, see https://blog.golang.org/pipelines for more info. - // goroutines send rooms to this channel - roomCh := make(chan gomatrixserverlib.PublicRoom, int(limit)) - // signalling channel to tell goroutines to stop sending rooms and quit - done := make(chan bool) - // signalling to say when we can close the room channel - var wg sync.WaitGroup - wg.Add(len(homeservers)) - // concurrently query for public rooms - for _, hs := range homeservers { - go func(homeserverDomain string) { - defer wg.Done() - util.GetLogger(ctx).WithField("hs", homeserverDomain).Info("Querying HS for public rooms") - fres, err := fedClient.GetPublicRooms(ctx, gomatrixserverlib.ServerName(homeserverDomain), int(limit), "", false, "") - if err != nil { - util.GetLogger(ctx).WithError(err).WithField("hs", homeserverDomain).Warn( - "bulkFetchPublicRoomsFromServers: failed to query hs", - ) - return - } - for _, room := range fres.Chunk { - // atomically send a room or stop - select { - case roomCh <- room: - case <-done: - util.GetLogger(ctx).WithError(err).WithField("hs", homeserverDomain).Info("Interrupted whilst sending rooms") - return - } - } - }(hs) - } - - // Close the room channel when the goroutines have quit so we don't leak, but don't let it stop the in-flight request. - // This also allows the request to fail fast if all HSes experience errors as it will cause the room channel to be - // closed. - go func() { - wg.Wait() - util.GetLogger(ctx).Info("Cleaning up resources") - close(roomCh) - }() - - // fan-in results with timeout. We stop when we reach the limit. -FanIn: - for len(publicRooms) < int(limit) || limit == 0 { - // add a room or timeout - select { - case room, ok := <-roomCh: - if !ok { - util.GetLogger(ctx).Info("All homeservers have been queried, returning results.") - break FanIn - } - publicRooms = append(publicRooms, room) - case <-time.After(15 * time.Second): // we've waited long enough, let's tell the client what we got. - util.GetLogger(ctx).Info("Waited 15s for federated public rooms, returning early") - break FanIn - case <-ctx.Done(): // the client hung up on us, let's stop. - util.GetLogger(ctx).Info("Client hung up, returning early") - break FanIn - } - } - // tell goroutines to stop - close(done) - - return publicRooms -} diff --git a/go.mod b/go.mod index a9774c2c8..ba222ed8f 100644 --- a/go.mod +++ b/go.mod @@ -26,22 +26,14 @@ require ( github.com/hashicorp/golang-lru v0.5.4 github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494 // indirect github.com/lib/pq v1.10.5 - github.com/libp2p/go-libp2p v0.13.0 - github.com/libp2p/go-libp2p-circuit v0.4.0 - github.com/libp2p/go-libp2p-core v0.8.3 - github.com/libp2p/go-libp2p-gostream v0.3.1 - github.com/libp2p/go-libp2p-http v0.2.0 - github.com/libp2p/go-libp2p-kad-dht v0.11.1 - github.com/libp2p/go-libp2p-pubsub v0.4.1 - github.com/libp2p/go-libp2p-record v0.1.3 github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e - github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 github.com/matrix-org/gomatrixserverlib v0.0.0-20220408160933-cf558306b56f github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.10 + github.com/miekg/dns v1.1.31 // indirect github.com/nats-io/nats-server/v2 v2.7.4-0.20220309205833-773636c1c5bb github.com/nats-io/nats.go v1.14.0 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 diff --git a/go.sum b/go.sum index ee4096fdc..8bb306a82 100644 --- a/go.sum +++ b/go.sum @@ -47,7 +47,6 @@ dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1 dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg= -github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/Arceliar/ironwood v0.0.0-20211125050254-8951369625d0 h1:QUqcb7BOcBU2p7Nax7pESOb8hrZYtI0Ts6j4v4mvcQo= github.com/Arceliar/ironwood v0.0.0-20211125050254-8951369625d0/go.mod h1:RP72rucOFm5udrnEzTmIWLRVGQiV/fSUAQXJ0RST/nk= @@ -80,7 +79,6 @@ github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= -github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/MFAshby/stdemuxerhook v1.0.0 h1:1XFGzakrsHMv76AeanPDL26NOgwjPl/OUxbGhJthwMc= github.com/MFAshby/stdemuxerhook v1.0.0/go.mod h1:nLMI9FUf9Hz98n+yAXsTMUR4RZQy28uCTLG1Fzvj/uY= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= @@ -122,7 +120,6 @@ github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqR github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= -github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/albertorestifo/dijkstra v0.0.0-20160910063646-aba76f725f72/go.mod h1:o+JdB7VetTHjLhU0N57x18B9voDBQe0paApdEAEoEfw= @@ -153,8 +150,6 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= -github.com/benbjohnson/clock v1.0.2/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= -github.com/benbjohnson/clock v1.0.3 h1:vkLuvpK4fmtSCuo60+yC63p7y0BmQ8gm5ZXGuBCJyXg= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -174,19 +169,6 @@ github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2w github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8= github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og= github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= -github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= -github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= -github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= -github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= -github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= -github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= -github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= -github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= -github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= -github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= -github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= @@ -340,7 +322,6 @@ github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmeka github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -363,21 +344,13 @@ github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1S github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= -github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018 h1:6xT9KW8zLC5IlbaIF5Q7JNieBoACT7iW0YTxQHR0in0= -github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= -github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= -github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= -github.com/dgraph-io/badger v1.6.1/go.mod h1:FRmFw3uxvcpa8zG3Rxs0th+hCLIuaQg8HlNV5bjgnuU= -github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= @@ -426,8 +399,6 @@ github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGE github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= -github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as= -github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -532,7 +503,6 @@ github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -546,7 +516,6 @@ github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3K github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= @@ -593,9 +562,6 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= -github.com/google/gopacket v1.1.18 h1:lum7VRA9kdlvBi7/v2p7/zcbkduHaCH/SVVyurs7OpY= -github.com/google/gopacket v1.1.18/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -646,8 +612,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpg github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= -github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= @@ -656,15 +620,12 @@ github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBt github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= @@ -686,9 +647,6 @@ github.com/hjson/hjson-go v3.1.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCe github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk= github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= -github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= -github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= -github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -699,49 +657,6 @@ github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ= -github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= -github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= -github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= -github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= -github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= -github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= -github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY= -github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= -github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= -github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= -github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= -github.com/ipfs/go-datastore v0.4.0/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= -github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= -github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= -github.com/ipfs/go-datastore v0.4.5 h1:cwOUcGMLdLPWgu3SlrCckCMznaGADbPqE0r8h768/Dg= -github.com/ipfs/go-datastore v0.4.5/go.mod h1:eXTcaaiN6uOlVCLS9GjJUJtlvJfM3xk23w3fyfrmmJs= -github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= -github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= -github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= -github.com/ipfs/go-ds-badger v0.0.5/go.mod h1:g5AuuCGmr7efyzQhLL8MzwqcauPojGPUaHzfGTzuE3s= -github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= -github.com/ipfs/go-ds-badger v0.2.1/go.mod h1:Tx7l3aTph3FMFrRS838dcSJh+jjA7cX9DrGVwx/NOwE= -github.com/ipfs/go-ds-badger v0.2.3/go.mod h1:pEYw0rgg3FIrywKKnL+Snr+w/LjJZVMTBRn4FS6UHUk= -github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= -github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= -github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= -github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= -github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= -github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= -github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= -github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= -github.com/ipfs/go-ipns v0.0.2 h1:oq4ErrV4hNQ2Eim257RTYRgfOSV/s8BDaf9iIl4NwFs= -github.com/ipfs/go-ipns v0.0.2/go.mod h1:WChil4e0/m9cIINWLxZe1Jtf77oz5L05rO2ei/uKJ5U= -github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= -github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk= -github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A= -github.com/ipfs/go-log v1.0.4 h1:6nLQdX4W8P9yZZFH7mO+X/PzjN8Laozm/lMJ6esdgzY= -github.com/ipfs/go-log v1.0.4/go.mod h1:oDCg2FkjogeFOhqqb+N39l2RpTNPL6F/StPkB3kPgcs= -github.com/ipfs/go-log/v2 v2.0.2/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= -github.com/ipfs/go-log/v2 v2.0.3/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= -github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= -github.com/ipfs/go-log/v2 v2.1.1 h1:G4TtqN+V9y9HY9TA6BwbCVyyBZ2B9MbCjR2MtGx8FR0= -github.com/ipfs/go-log/v2 v2.1.1/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk= @@ -749,23 +664,7 @@ github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0Gqw github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbBFOsPw= -github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= -github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= -github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= -github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= -github.com/jbenet/go-cienv v0.0.0-20150120210510-1bb1476777ec/go.mod h1:rGaEvXB4uRSZMmzKNLoXvTu1sfx+1kv/DojUlPrSZGs= -github.com/jbenet/go-cienv v0.1.0 h1:Vc/s0QbQtoxX8MwwSLWWh+xNNZvM3Lw7NsTcHrvvhMc= -github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= -github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2/go.mod h1:8GXXJV31xl8whumTzdZsTt3RnUIiPqzkyf7mxToRCMs= -github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk= -github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= -github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= -github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= -github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= -github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= -github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= @@ -773,7 +672,6 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22 github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -823,7 +721,6 @@ github.com/julienschmidt/httprouter v1.1.1-0.20151013225520-77a895ad01eb/go.mod github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= -github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/kardianos/minwinsvc v1.0.0/go.mod h1:Bgd0oc+D0Qo3bBytmNtyRKVlp85dAloLKhfxanPFFRc= github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8= github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE= @@ -834,7 +731,6 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= @@ -846,8 +742,6 @@ github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgo github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d h1:68u9r4wEvL3gYg2jvAOgROwZ3H+Y3hIDk4tbbmIjcYQ= -github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -866,215 +760,6 @@ github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v1.10.5 h1:J+gdV2cUmX7ZqL2B0lFcW0m+egaHC2V3lpO8nWxyYiQ= github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= -github.com/libp2p/go-addr-util v0.0.2 h1:7cWK5cdA5x72jX0g8iLrQWm5TRJZ6CzGdPEhWj7plWU= -github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E= -github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= -github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs= -github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= -github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= -github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= -github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc= -github.com/libp2p/go-conn-security-multistream v0.2.0 h1:uNiDjS58vrvJTg9jO6bySd1rMKejieG7v45ekqHbZ1M= -github.com/libp2p/go-conn-security-multistream v0.2.0/go.mod h1:hZN4MjlNetKD3Rq5Jb/P5ohUnFLNzEAR4DLSzpn2QLU= -github.com/libp2p/go-eventbus v0.1.0/go.mod h1:vROgu5cs5T7cv7POWlWxBaVLxfSegC5UGQf8A2eEmx4= -github.com/libp2p/go-eventbus v0.2.1 h1:VanAdErQnpTioN2TowqNcOijf6YwhuODe4pPKSDpxGc= -github.com/libp2p/go-eventbus v0.2.1/go.mod h1:jc2S4SoEVPP48H9Wpzm5aiGwUCBMfGhVhhBjyhhCJs8= -github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8= -github.com/libp2p/go-flow-metrics v0.0.2/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= -github.com/libp2p/go-flow-metrics v0.0.3 h1:8tAs/hSdNvUiLgtlSy3mxwxWP4I9y/jlkPFT7epKdeM= -github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= -github.com/libp2p/go-libp2p v0.6.1/go.mod h1:CTFnWXogryAHjXAKEbOf1OWY+VeAP3lDMZkfEI5sT54= -github.com/libp2p/go-libp2p v0.7.0/go.mod h1:hZJf8txWeCduQRDC/WSqBGMxaTHCOYHt2xSU1ivxn0k= -github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniVO7zIHGMw= -github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qDKwXujH5o= -github.com/libp2p/go-libp2p v0.12.0/go.mod h1:FpHZrfC1q7nA8jitvdjKBDF31hguaC676g/nT9PgQM0= -github.com/libp2p/go-libp2p v0.13.0 h1:tDdrXARSghmusdm0nf1U/4M8aj8Rr0V2IzQOXmbzQ3s= -github.com/libp2p/go-libp2p v0.13.0/go.mod h1:pM0beYdACRfHO1WcJlp65WXyG2A6NqYM+t2DTVAJxMo= -github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052 h1:BM7aaOF7RpmNn9+9g6uTjGJ0cTzWr5j9i9IKeun2M8U= -github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052/go.mod h1:nRMRTab+kZuk0LnKZpxhOVH/ndsdr2Nr//Zltc/vwgo= -github.com/libp2p/go-libp2p-autonat v0.1.1/go.mod h1:OXqkeGOY2xJVWKAGV2inNF5aKN/djNA3fdpCWloIudE= -github.com/libp2p/go-libp2p-autonat v0.2.0/go.mod h1:DX+9teU4pEEoZUqR1PiMlqliONQdNbfzE1C718tcViI= -github.com/libp2p/go-libp2p-autonat v0.2.1/go.mod h1:MWtAhV5Ko1l6QBsHQNSuM6b1sRkXrpk0/LqCr+vCVxI= -github.com/libp2p/go-libp2p-autonat v0.2.2/go.mod h1:HsM62HkqZmHR2k1xgX34WuWDzk/nBwNHoeyyT4IWV6A= -github.com/libp2p/go-libp2p-autonat v0.4.0 h1:3y8XQbpr+ssX8QfZUHekjHCYK64sj6/4hnf/awD4+Ug= -github.com/libp2p/go-libp2p-autonat v0.4.0/go.mod h1:YxaJlpr81FhdOv3W3BTconZPfhaYivRdf53g+S2wobk= -github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro= -github.com/libp2p/go-libp2p-blankhost v0.1.4/go.mod h1:oJF0saYsAXQCSfDq254GMNmLNz6ZTHTOvtF4ZydUvwU= -github.com/libp2p/go-libp2p-blankhost v0.2.0 h1:3EsGAi0CBGcZ33GwRuXEYJLLPoVWyXJ1bcJzAJjINkk= -github.com/libp2p/go-libp2p-blankhost v0.2.0/go.mod h1:eduNKXGTioTuQAUcZ5epXi9vMl+t4d8ugUBRQ4SqaNQ= -github.com/libp2p/go-libp2p-circuit v0.1.4/go.mod h1:CY67BrEjKNDhdTk8UgBX1Y/H5c3xkAcs3gnksxY7osU= -github.com/libp2p/go-libp2p-circuit v0.2.1/go.mod h1:BXPwYDN5A8z4OEY9sOfr2DUQMLQvKt/6oku45YUmjIo= -github.com/libp2p/go-libp2p-circuit v0.4.0 h1:eqQ3sEYkGTtybWgr6JLqJY6QLtPWRErvFjFDfAOO1wc= -github.com/libp2p/go-libp2p-circuit v0.4.0/go.mod h1:t/ktoFIUzM6uLQ+o1G6NuBl2ANhBKN9Bc8jRIk31MoA= -github.com/libp2p/go-libp2p-connmgr v0.2.4 h1:TMS0vc0TCBomtQJyWr7fYxcVYYhx+q/2gF++G5Jkl/w= -github.com/libp2p/go-libp2p-connmgr v0.2.4/go.mod h1:YV0b/RIm8NGPnnNWM7hG9Q38OeQiQfKhHCCs1++ufn0= -github.com/libp2p/go-libp2p-core v0.0.1/go.mod h1:g/VxnTZ/1ygHxH3dKok7Vno1VfpvGcGip57wjTU4fco= -github.com/libp2p/go-libp2p-core v0.0.4/go.mod h1:jyuCQP356gzfCFtRKyvAbNkyeuxb7OlyhWZ3nls5d2I= -github.com/libp2p/go-libp2p-core v0.2.0/go.mod h1:X0eyB0Gy93v0DZtSYbEM7RnMChm9Uv3j7yRXjO77xSI= -github.com/libp2p/go-libp2p-core v0.2.2/go.mod h1:8fcwTbsG2B+lTgRJ1ICZtiM5GWCWZVoVrLaDRvIRng0= -github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= -github.com/libp2p/go-libp2p-core v0.2.5/go.mod h1:6+5zJmKhsf7yHn1RbmYDu08qDUpIUxGdqHuEZckmZOA= -github.com/libp2p/go-libp2p-core v0.3.0/go.mod h1:ACp3DmS3/N64c2jDzcV429ukDpicbL6+TrrxANBjPGw= -github.com/libp2p/go-libp2p-core v0.3.1/go.mod h1:thvWy0hvaSBhnVBaW37BvzgVV68OUhgJJLAa6almrII= -github.com/libp2p/go-libp2p-core v0.4.0/go.mod h1:49XGI+kc38oGVwqSBhDEwytaAxgZasHhFfQKibzTls0= -github.com/libp2p/go-libp2p-core v0.5.0/go.mod h1:49XGI+kc38oGVwqSBhDEwytaAxgZasHhFfQKibzTls0= -github.com/libp2p/go-libp2p-core v0.5.1/go.mod h1:uN7L2D4EvPCvzSH5SrhR72UWbnSGpt5/a35Sm4upn4Y= -github.com/libp2p/go-libp2p-core v0.5.3/go.mod h1:uN7L2D4EvPCvzSH5SrhR72UWbnSGpt5/a35Sm4upn4Y= -github.com/libp2p/go-libp2p-core v0.5.4/go.mod h1:uN7L2D4EvPCvzSH5SrhR72UWbnSGpt5/a35Sm4upn4Y= -github.com/libp2p/go-libp2p-core v0.5.5/go.mod h1:vj3awlOr9+GMZJFH9s4mpt9RHHgGqeHCopzbYKZdRjM= -github.com/libp2p/go-libp2p-core v0.5.6/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX0bJvM49Ykaswo= -github.com/libp2p/go-libp2p-core v0.5.7/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX0bJvM49Ykaswo= -github.com/libp2p/go-libp2p-core v0.6.0/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX0bJvM49Ykaswo= -github.com/libp2p/go-libp2p-core v0.6.1/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= -github.com/libp2p/go-libp2p-core v0.7.0/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= -github.com/libp2p/go-libp2p-core v0.8.0/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= -github.com/libp2p/go-libp2p-core v0.8.3 h1:BZTReEF6o8g/n4DwxTyeFannOeae35Xy0TD+mES3CNE= -github.com/libp2p/go-libp2p-core v0.8.3/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= -github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= -github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg= -github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQOu38Fu7LJGEOK2gQltw= -github.com/libp2p/go-libp2p-discovery v0.5.0 h1:Qfl+e5+lfDgwdrXdu4YNCWyEo3fWuP+WgN9mN0iWviQ= -github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug= -github.com/libp2p/go-libp2p-gostream v0.3.0/go.mod h1:pLBQu8db7vBMNINGsAwLL/ZCE8wng5V1FThoaE5rNjc= -github.com/libp2p/go-libp2p-gostream v0.3.1 h1:XlwohsPn6uopGluEWs1Csv1QCEjrTXf2ZQagzZ5paAg= -github.com/libp2p/go-libp2p-gostream v0.3.1/go.mod h1:1V3b+u4Zhaq407UUY9JLCpboaeufAeVQbnvAt12LRsI= -github.com/libp2p/go-libp2p-http v0.2.0 h1:GYeVd+RZzkRa8XFLITqOpcrIQG6KbFLPJqII6HHBHzY= -github.com/libp2p/go-libp2p-http v0.2.0/go.mod h1:GlNKFqDZHe25LVy2CvnZKx75/jLtMaD3VxZV6N39X7E= -github.com/libp2p/go-libp2p-kad-dht v0.11.1 h1:FsriVQhOUZpCotWIjyFSjEDNJmUzuMma/RyyTDZanwc= -github.com/libp2p/go-libp2p-kad-dht v0.11.1/go.mod h1:5ojtR2acDPqh/jXf5orWy8YGb8bHQDS+qeDcoscL/PI= -github.com/libp2p/go-libp2p-kbucket v0.4.7 h1:spZAcgxifvFZHBD8tErvppbnNiKA5uokDu3CV7axu70= -github.com/libp2p/go-libp2p-kbucket v0.4.7/go.mod h1:XyVo99AfQH0foSf176k4jY1xUJ2+jUJIZCSDm7r2YKk= -github.com/libp2p/go-libp2p-loggables v0.1.0 h1:h3w8QFfCt2UJl/0/NW4K829HX/0S4KD31PQ7m8UXXO8= -github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= -github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= -github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiYdAWNYHrwImKLnE= -github.com/libp2p/go-libp2p-mplex v0.2.2/go.mod h1:74S9eum0tVQdAfFiKxAyKzNdSuLqw5oadDq7+L/FELo= -github.com/libp2p/go-libp2p-mplex v0.2.3/go.mod h1:CK3p2+9qH9x+7ER/gWWDYJ3QW5ZxWDkm+dVvjfuG3ek= -github.com/libp2p/go-libp2p-mplex v0.3.0/go.mod h1:l9QWxRbbb5/hQMECEb908GbS9Sm2UAR2KFZKUJEynEs= -github.com/libp2p/go-libp2p-mplex v0.4.0/go.mod h1:yCyWJE2sc6TBTnFpjvLuEJgTSw/u+MamvzILKdX7asw= -github.com/libp2p/go-libp2p-mplex v0.4.1 h1:/pyhkP1nLwjG3OM+VuaNJkQT/Pqq73WzB3aDN3Fx1sc= -github.com/libp2p/go-libp2p-mplex v0.4.1/go.mod h1:cmy+3GfqfM1PceHTLL7zQzAAYaryDu6iPSC+CIb094g= -github.com/libp2p/go-libp2p-nat v0.0.5/go.mod h1:1qubaE5bTZMJE+E/uu2URroMbzdubFz1ChgiN79yKPE= -github.com/libp2p/go-libp2p-nat v0.0.6 h1:wMWis3kYynCbHoyKLPBEMu4YRLltbm8Mk08HGSfvTkU= -github.com/libp2p/go-libp2p-nat v0.0.6/go.mod h1:iV59LVhB3IkFvS6S6sauVTSOrNEANnINbI/fkaLimiw= -github.com/libp2p/go-libp2p-netutil v0.1.0 h1:zscYDNVEcGxyUpMd0JReUZTrpMfia8PmLKcKF72EAMQ= -github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU= -github.com/libp2p/go-libp2p-noise v0.1.1 h1:vqYQWvnIcHpIoWJKC7Al4D6Hgj0H012TuXRhPwSMGpQ= -github.com/libp2p/go-libp2p-noise v0.1.1/go.mod h1:QDFLdKX7nluB7DEnlVPbz7xlLHdwHFA9HiohJRr3vwM= -github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY= -github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY= -github.com/libp2p/go-libp2p-peerstore v0.1.3/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI= -github.com/libp2p/go-libp2p-peerstore v0.1.4/go.mod h1:+4BDbDiiKf4PzpANZDAT+knVdLxvqh7hXOujessqdzs= -github.com/libp2p/go-libp2p-peerstore v0.2.0/go.mod h1:N2l3eVIeAitSg3Pi2ipSrJYnqhVnMNQZo9nkSCuAbnQ= -github.com/libp2p/go-libp2p-peerstore v0.2.1/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRjwRLBr4TYKfNgrUkOPA= -github.com/libp2p/go-libp2p-peerstore v0.2.2/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRjwRLBr4TYKfNgrUkOPA= -github.com/libp2p/go-libp2p-peerstore v0.2.6 h1:2ACefBX23iMdJU9Ke+dcXt3w86MIryes9v7In4+Qq3U= -github.com/libp2p/go-libp2p-peerstore v0.2.6/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= -github.com/libp2p/go-libp2p-pnet v0.2.0 h1:J6htxttBipJujEjz1y0a5+eYoiPcFHhSYHH6na5f0/k= -github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYcO0BW4wssv21LA= -github.com/libp2p/go-libp2p-pubsub v0.4.1 h1:j4umIg5nyus+sqNfU+FWvb9aeYFQH/A+nDFhWj+8yy8= -github.com/libp2p/go-libp2p-pubsub v0.4.1/go.mod h1:izkeMLvz6Ht8yAISXjx60XUQZMq9ZMe5h2ih4dLIBIQ= -github.com/libp2p/go-libp2p-record v0.1.2/go.mod h1:pal0eNcT5nqZaTV7UGhqeGqxFgGdsU/9W//C8dqjQDk= -github.com/libp2p/go-libp2p-record v0.1.3 h1:R27hoScIhQf/A8XJZ8lYpnqh9LatJ5YbHs28kCIfql0= -github.com/libp2p/go-libp2p-record v0.1.3/go.mod h1:yNUff/adKIfPnYQXgp6FQmNu3gLJ6EMg7+/vv2+9pY4= -github.com/libp2p/go-libp2p-routing-helpers v0.2.3/go.mod h1:795bh+9YeoFl99rMASoiVgHdi5bjack0N1+AFAdbvBw= -github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= -github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= -github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8= -github.com/libp2p/go-libp2p-secio v0.2.2/go.mod h1:wP3bS+m5AUnFA+OFO7Er03uO1mncHG0uVwGrwvjYlNY= -github.com/libp2p/go-libp2p-swarm v0.1.0/go.mod h1:wQVsCdjsuZoc730CgOvh5ox6K8evllckjebkdiY5ta4= -github.com/libp2p/go-libp2p-swarm v0.2.2/go.mod h1:fvmtQ0T1nErXym1/aa1uJEyN7JzaTNyBcHImCxRpPKU= -github.com/libp2p/go-libp2p-swarm v0.2.3/go.mod h1:P2VO/EpxRyDxtChXz/VPVXyTnszHvokHKRhfkEgFKNM= -github.com/libp2p/go-libp2p-swarm v0.2.8/go.mod h1:JQKMGSth4SMqonruY0a8yjlPVIkb0mdNSwckW7OYziM= -github.com/libp2p/go-libp2p-swarm v0.3.0/go.mod h1:hdv95GWCTmzkgeJpP+GK/9D9puJegb7H57B5hWQR5Kk= -github.com/libp2p/go-libp2p-swarm v0.3.1/go.mod h1:hdv95GWCTmzkgeJpP+GK/9D9puJegb7H57B5hWQR5Kk= -github.com/libp2p/go-libp2p-swarm v0.4.0 h1:hahq/ijRoeH6dgROOM8x7SeaKK5VgjjIr96vdrT+NUA= -github.com/libp2p/go-libp2p-swarm v0.4.0/go.mod h1:XVFcO52VoLoo0eitSxNQWYq4D6sydGOweTOAjJNraCw= -github.com/libp2p/go-libp2p-testing v0.0.2/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= -github.com/libp2p/go-libp2p-testing v0.0.3/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= -github.com/libp2p/go-libp2p-testing v0.0.4/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= -github.com/libp2p/go-libp2p-testing v0.1.0/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eqnjVyavGroI2nxEM0= -github.com/libp2p/go-libp2p-testing v0.1.1/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eqnjVyavGroI2nxEM0= -github.com/libp2p/go-libp2p-testing v0.1.2-0.20200422005655-8775583591d8/go.mod h1:Qy8sAncLKpwXtS2dSnDOP8ktexIAHKu+J+pnZOFZLTc= -github.com/libp2p/go-libp2p-testing v0.3.0/go.mod h1:efZkql4UZ7OVsEfaxNHZPzIehtsBXMrXnCfJIgDti5g= -github.com/libp2p/go-libp2p-testing v0.4.0 h1:PrwHRi0IGqOwVQWR3xzgigSlhlLfxgfXgkHxr77EghQ= -github.com/libp2p/go-libp2p-testing v0.4.0/go.mod h1:Q+PFXYoiYFN5CAEG2w3gLPEzotlKsNSbKQ/lImlOWF0= -github.com/libp2p/go-libp2p-tls v0.1.3 h1:twKMhMu44jQO+HgQK9X8NHO5HkeJu2QbhLzLJpa8oNM= -github.com/libp2p/go-libp2p-tls v0.1.3/go.mod h1:wZfuewxOndz5RTnCAxFliGjvYSDA40sKitV4c50uI1M= -github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA= -github.com/libp2p/go-libp2p-transport-upgrader v0.2.0/go.mod h1:mQcrHj4asu6ArfSoMuyojOdjx73Q47cYD7s5+gZOlns= -github.com/libp2p/go-libp2p-transport-upgrader v0.3.0/go.mod h1:i+SKzbRnvXdVbU3D1dwydnTmKRPXiAR/fyvi1dXuL4o= -github.com/libp2p/go-libp2p-transport-upgrader v0.4.0 h1:xwj4h3hJdBrxqMOyMUjwscjoVst0AASTsKtZiTChoHI= -github.com/libp2p/go-libp2p-transport-upgrader v0.4.0/go.mod h1:J4ko0ObtZSmgn5BX5AmegP+dK3CSnU2lMCKsSq/EY0s= -github.com/libp2p/go-libp2p-yamux v0.2.0/go.mod h1:Db2gU+XfLpm6E4rG5uGCFX6uXA8MEXOxFcRoXUODaK8= -github.com/libp2p/go-libp2p-yamux v0.2.2/go.mod h1:lIohaR0pT6mOt0AZ0L2dFze9hds9Req3OfS+B+dv4qw= -github.com/libp2p/go-libp2p-yamux v0.2.5/go.mod h1:Zpgj6arbyQrmZ3wxSZxfBmbdnWtbZ48OpsfmQVTErwA= -github.com/libp2p/go-libp2p-yamux v0.2.7/go.mod h1:X28ENrBMU/nm4I3Nx4sZ4dgjZ6VhLEn0XhIoZ5viCwU= -github.com/libp2p/go-libp2p-yamux v0.2.8/go.mod h1:/t6tDqeuZf0INZMTgd0WxIRbtK2EzI2h7HbFm9eAKI4= -github.com/libp2p/go-libp2p-yamux v0.4.0/go.mod h1:+DWDjtFMzoAwYLVkNZftoucn7PelNoy5nm3tZ3/Zw30= -github.com/libp2p/go-libp2p-yamux v0.5.0/go.mod h1:AyR8k5EzyM2QN9Bbdg6X1SkVVuqLwTGf0L4DFq9g6po= -github.com/libp2p/go-libp2p-yamux v0.5.1 h1:sX4WQPHMhRxJE5UZTfjEuBvlQWXB5Bo3A2JK9ZJ9EM0= -github.com/libp2p/go-libp2p-yamux v0.5.1/go.mod h1:dowuvDu8CRWmr0iqySMiSxK+W0iL5cMVO9S94Y6gkv4= -github.com/libp2p/go-maddr-filter v0.0.4/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q= -github.com/libp2p/go-maddr-filter v0.0.5/go.mod h1:Jk+36PMfIqCJhAnaASRH83bdAvfDRp/w6ENFaC9bG+M= -github.com/libp2p/go-maddr-filter v0.1.0/go.mod h1:VzZhTXkMucEGGEOSKddrwGiOv0tUhgnKqNEmIAz/bPU= -github.com/libp2p/go-mplex v0.0.3/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0= -github.com/libp2p/go-mplex v0.1.0/go.mod h1:SXgmdki2kwCUlCCbfGLEgHjC4pFqhTp0ZoV6aiKgxDU= -github.com/libp2p/go-mplex v0.1.1/go.mod h1:Xgz2RDCi3co0LeZfgjm4OgUF15+sVR8SRcu3SFXI1lk= -github.com/libp2p/go-mplex v0.1.2/go.mod h1:Xgz2RDCi3co0LeZfgjm4OgUF15+sVR8SRcu3SFXI1lk= -github.com/libp2p/go-mplex v0.2.0/go.mod h1:0Oy/A9PQlwBytDRp4wSkFnzHYDKcpLot35JQ6msjvYQ= -github.com/libp2p/go-mplex v0.3.0 h1:U1T+vmCYJaEoDJPV1aq31N56hS+lJgb397GsylNSgrU= -github.com/libp2p/go-mplex v0.3.0/go.mod h1:0Oy/A9PQlwBytDRp4wSkFnzHYDKcpLot35JQ6msjvYQ= -github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= -github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= -github.com/libp2p/go-msgio v0.0.6 h1:lQ7Uc0kS1wb1EfRxO2Eir/RJoHkHn7t6o+EiwsYIKJA= -github.com/libp2p/go-msgio v0.0.6/go.mod h1:4ecVB6d9f4BDSL5fqvPiC4A3KivjWn+Venn/1ALLMWA= -github.com/libp2p/go-nat v0.0.4/go.mod h1:Nmw50VAvKuk38jUBcmNh6p9lUJLoODbJRvYAa/+KSDo= -github.com/libp2p/go-nat v0.0.5 h1:qxnwkco8RLKqVh1NmjQ+tJ8p8khNLFxuElYG/TwqW4Q= -github.com/libp2p/go-nat v0.0.5/go.mod h1:B7NxsVNPZmRLvMOwiEO1scOSyjA56zxYAGv1yQgRkEU= -github.com/libp2p/go-netroute v0.1.2/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= -github.com/libp2p/go-netroute v0.1.3 h1:1ngWRx61us/EpaKkdqkMjKk/ufr/JlIFYQAxV2XX8Ig= -github.com/libp2p/go-netroute v0.1.3/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= -github.com/libp2p/go-openssl v0.0.2/go.mod h1:v8Zw2ijCSWBQi8Pq5GAixw6DbFfa9u6VIYDXnvOXkc0= -github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= -github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= -github.com/libp2p/go-openssl v0.0.5/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= -github.com/libp2p/go-openssl v0.0.7 h1:eCAzdLejcNVBzP/iZM9vqHnQm+XyCEbSSIheIPRGNsw= -github.com/libp2p/go-openssl v0.0.7/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= -github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= -github.com/libp2p/go-reuseport v0.0.2 h1:XSG94b1FJfGA01BUrT82imejHQyTxO4jEWqheyCXYvU= -github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQyMVAheVBD9nQ= -github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= -github.com/libp2p/go-reuseport-transport v0.0.3/go.mod h1:Spv+MPft1exxARzP2Sruj2Wb5JSyHNncjf1Oi2dEbzM= -github.com/libp2p/go-reuseport-transport v0.0.4 h1:OZGz0RB620QDGpv300n1zaOcKGGAoGVf8h9txtt/1uM= -github.com/libp2p/go-reuseport-transport v0.0.4/go.mod h1:trPa7r/7TJK/d+0hdBLOCGvpQQVOU74OXbNCIMkufGw= -github.com/libp2p/go-sockaddr v0.0.2 h1:tCuXfpA9rq7llM/v834RKc/Xvovy/AqM9kHvTV/jY/Q= -github.com/libp2p/go-sockaddr v0.0.2/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k= -github.com/libp2p/go-stream-muxer v0.0.1/go.mod h1:bAo8x7YkSpadMTbtTaxGVHWUQsR/l5MEaHbKaliuT14= -github.com/libp2p/go-stream-muxer-multistream v0.2.0/go.mod h1:j9eyPol/LLRqT+GPLSxvimPhNph4sfYfMoDPd7HkzIc= -github.com/libp2p/go-stream-muxer-multistream v0.3.0 h1:TqnSHPJEIqDEO7h1wZZ0p3DXdvDSiLHQidKKUGZtiOY= -github.com/libp2p/go-stream-muxer-multistream v0.3.0/go.mod h1:yDh8abSIzmZtqtOt64gFJUXEryejzNb0lisTt+fAMJA= -github.com/libp2p/go-tcp-transport v0.1.0/go.mod h1:oJ8I5VXryj493DEJ7OsBieu8fcg2nHGctwtInJVpipc= -github.com/libp2p/go-tcp-transport v0.1.1/go.mod h1:3HzGvLbx6etZjnFlERyakbaYPdfjg2pWP97dFZworkY= -github.com/libp2p/go-tcp-transport v0.2.0/go.mod h1:vX2U0CnWimU4h0SGSEsg++AzvBcroCGYw28kh94oLe0= -github.com/libp2p/go-tcp-transport v0.2.1 h1:ExZiVQV+h+qL16fzCWtd1HSzPsqWottJ8KXwWaVi8Ns= -github.com/libp2p/go-tcp-transport v0.2.1/go.mod h1:zskiJ70MEfWz2MKxvFB/Pv+tPIB1PpPUrHIWQ8aFw7M= -github.com/libp2p/go-ws-transport v0.2.0/go.mod h1:9BHJz/4Q5A9ludYWKoGCFC5gUElzlHoKzu0yY9p/klM= -github.com/libp2p/go-ws-transport v0.3.0/go.mod h1:bpgTJmRZAvVHrgHybCVyqoBmyLQ1fiZuEaBYusP5zsk= -github.com/libp2p/go-ws-transport v0.3.1/go.mod h1:bpgTJmRZAvVHrgHybCVyqoBmyLQ1fiZuEaBYusP5zsk= -github.com/libp2p/go-ws-transport v0.4.0 h1:9tvtQ9xbws6cA5LvqdE6Ne3vcmGB4f1z9SByggk4s0k= -github.com/libp2p/go-ws-transport v0.4.0/go.mod h1:EcIEKqf/7GDjth6ksuS/6p7R49V4CBY6/E7R/iyhYUA= -github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= -github.com/libp2p/go-yamux v1.3.0/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= -github.com/libp2p/go-yamux v1.3.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= -github.com/libp2p/go-yamux v1.3.5/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= -github.com/libp2p/go-yamux v1.3.7/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= -github.com/libp2p/go-yamux v1.4.0/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= -github.com/libp2p/go-yamux v1.4.1 h1:P1Fe9vF4th5JOxxgQvfbOHkrGqIZniTLf+ddhZp8YTI= -github.com/libp2p/go-yamux v1.4.1/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= -github.com/libp2p/go-yamux/v2 v2.0.0 h1:vSGhAy5u6iHBq11ZDcyHH4Blcf9xlBhT4WQDoOE90LU= -github.com/libp2p/go-yamux/v2 v2.0.0/go.mod h1:NVWira5+sVUIU6tu1JWvaRn1dRnG+cawOJiflsAM+7U= github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= github.com/lucas-clemente/quic-go v0.26.0 h1:ALBQXr9UJ8A1LyzvceX4jd9QFsHvlI0RR6BkV16o00A= github.com/lucas-clemente/quic-go v0.26.0/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI= @@ -1085,7 +770,6 @@ github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -1105,8 +789,6 @@ github.com/masterzen/winrm v0.0.0-20161014151040-7a535cd943fc/go.mod h1:CfZSN7zw github.com/masterzen/xmlpath v0.0.0-20140218185901-13f4951698ad/go.mod h1:A0zPC53iKKKcXYxr4ROjpQRQ5FgJXtelNdSmHHuq/tY= github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e h1:DP5RC0Z3XdyBEW5dKt8YPeN6vZbm6OzVaGVp7f1BQRM= github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= -github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 h1:eqE5OnGx9ZMWmrRbD3KF/3KtTunw0iQulI7YxOIdxo4= -github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4/go.mod h1:3WluEZ9QXSwU30tWYqktnpC1x9mwZKx1r8uAv8Iq+a4= github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw2QV3YD/fRrzEDPNGgTlJlvXY0EHHnT87wF3OA= github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= @@ -1121,14 +803,12 @@ github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335M github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/mattn/go-colorable v0.0.6/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.0-20160806122752-66b8e73f3f5c/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -1150,26 +830,15 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182aff github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= -github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.25/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= -github.com/miekg/dns v1.1.28/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.31 h1:sJFOl9BgwbYAWOGEwr61FU28pqsBNdpRBnhGXtO06Oo= github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= -github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= -github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= -github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= -github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= -github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= -github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= -github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= -github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -1202,65 +871,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= -github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= -github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= -github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= -github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= -github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= -github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= -github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4= -github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= -github.com/multiformats/go-multiaddr v0.0.1/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= -github.com/multiformats/go-multiaddr v0.0.2/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= -github.com/multiformats/go-multiaddr v0.0.4/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= -github.com/multiformats/go-multiaddr v0.1.0/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= -github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= -github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= -github.com/multiformats/go-multiaddr v0.2.1/go.mod h1:s/Apk6IyxfvMjDafnhJgJ3/46z7tZ04iMk5wP4QMGGE= -github.com/multiformats/go-multiaddr v0.2.2/go.mod h1:NtfXiOtHvghW9KojvtySjH5y0u0xW5UouOmQQrn6a3Y= -github.com/multiformats/go-multiaddr v0.3.0/go.mod h1:dF9kph9wfJ+3VLAaeBqo9Of8x4fJxp6ggJGteB8HQTI= -github.com/multiformats/go-multiaddr v0.3.1 h1:1bxa+W7j9wZKTZREySx1vPMs2TqrYWjVZ7zE6/XLG1I= -github.com/multiformats/go-multiaddr v0.3.1/go.mod h1:uPbspcUPd5AfaP6ql3ujFY+QWzmBD8uLLL4bXW0XfGc= -github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= -github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= -github.com/multiformats/go-multiaddr-dns v0.2.0 h1:YWJoIDwLePniH7OU5hBnDZV6SWuvJqJ0YtN6pLeH9zA= -github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= -github.com/multiformats/go-multiaddr-fmt v0.0.1/go.mod h1:aBYjqL4T/7j4Qx+R73XSv/8JsgnRFlf0w2KGLCmXl3Q= -github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= -github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= -github.com/multiformats/go-multiaddr-net v0.0.1/go.mod h1:nw6HSxNmCIQH27XPGBuX+d1tnvM7ihcFwHMSstNAVUU= -github.com/multiformats/go-multiaddr-net v0.1.0/go.mod h1:5JNbcfBOP4dnhoZOv10JJVkJO0pCCEf8mTnipAo2UZQ= -github.com/multiformats/go-multiaddr-net v0.1.1/go.mod h1:5JNbcfBOP4dnhoZOv10JJVkJO0pCCEf8mTnipAo2UZQ= -github.com/multiformats/go-multiaddr-net v0.1.2/go.mod h1:QsWt3XK/3hwvNxZJp92iMQKME1qHfpYmyIjFVsSOY6Y= -github.com/multiformats/go-multiaddr-net v0.1.3/go.mod h1:ilNnaM9HbmVFqsb/qcNysjCu4PVONlrBZpHIrw/qQuA= -github.com/multiformats/go-multiaddr-net v0.1.4/go.mod h1:ilNnaM9HbmVFqsb/qcNysjCu4PVONlrBZpHIrw/qQuA= -github.com/multiformats/go-multiaddr-net v0.1.5/go.mod h1:ilNnaM9HbmVFqsb/qcNysjCu4PVONlrBZpHIrw/qQuA= -github.com/multiformats/go-multiaddr-net v0.2.0 h1:MSXRGN0mFymt6B1yo/6BPnIRpLPEnKgQNvVfCX5VDJk= -github.com/multiformats/go-multiaddr-net v0.2.0/go.mod h1:gGdH3UXny6U3cKKYCvpXI5rnK7YaOIEOPVDI9tsJbEA= -github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= -github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= -github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= -github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= -github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= -github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= -github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= -github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= -github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= -github.com/multiformats/go-multihash v0.0.14 h1:QoBceQYQQtNUuf6s7wHxnE2c8bhbMqhfGzNI032se/I= -github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= -github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= -github.com/multiformats/go-multistream v0.1.1/go.mod h1:KmHZ40hzVxiaiwlj3MEbYgK9JFk2/9UktWZAF54Du38= -github.com/multiformats/go-multistream v0.2.0 h1:6AuNmQVKUkRnddw2YiDjt5Elit40SFxMJkVnhmETXtU= -github.com/multiformats/go-multistream v0.2.0/go.mod h1:5GZPQZbkWOLOn3J2y4Y99vVW7vOfsAflxARk3x14o6k= -github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= -github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= -github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= -github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY= -github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -1296,7 +908,6 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -1310,7 +921,6 @@ github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= @@ -1349,7 +959,6 @@ github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqi github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= -github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= @@ -1475,17 +1084,11 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= -github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= -github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= -github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= -github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -1503,7 +1106,6 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -1524,7 +1126,6 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69 github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -1572,21 +1173,9 @@ github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmF github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= -github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= -github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= -github.com/whyrusleeping/go-logging v0.0.1/go.mod h1:lDPYj54zutzG1XYfHAhcc7oNXEburHQBn+Iqd4yS4vE= -github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvXwGvqIpk20FA= -github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9 h1:Y1/FEOpaCpD21WxrmfeIYCFPuVPRCY2XZTWzTNHGw30= -github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= -github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 h1:E9S12nwJwEOXe2d6gT6qxdvqMnNq+VnSsKPgm2ZZNds= -github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= -github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= -github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= -github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= @@ -1623,12 +1212,10 @@ go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvS go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= @@ -1653,28 +1240,17 @@ go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqe go.opentelemetry.io/proto/otlp v0.11.0/go.mod h1:QpEjXPrNQzrFDZgoTo49dgHR9RYRSrg3NAKnUGl9YpQ= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= -go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= -go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= -go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= -golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180214000028-650f4a345ab4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1683,22 +1259,16 @@ golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -1773,7 +1343,6 @@ golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -1858,7 +1427,6 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1871,19 +1439,15 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2015,7 +1579,6 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -2038,8 +1601,6 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -2204,7 +1765,6 @@ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= @@ -2269,8 +1829,6 @@ gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= -gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637/go.mod h1:BHsqpu/nsuzkT5BpiH1EMZPLyqSMM8JbIavyFACoFNk= From 0eb5bd1e13b974a6a1f9e4548eab979cd6fd7528 Mon Sep 17 00:00:00 2001 From: kegsay Date: Thu, 21 Apr 2022 17:17:52 +0100 Subject: [PATCH 05/50] Update AWSY test groups (#2365) * Update AWSY test groups * Better names --- are-we-synapse-yet.list | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/are-we-synapse-yet.list b/are-we-synapse-yet.list index 4281171ab..c776a7400 100644 --- a/are-we-synapse-yet.list +++ b/are-we-synapse-yet.list @@ -212,11 +212,12 @@ plv Users cannot set kick powerlevel higher than their own (2 subtests) plv Users cannot set redact powerlevel higher than their own (2 subtests) v1s Check that event streams started after a client joined a room work (SYT-1) v1s Event stream catches up fully after many messages -xxx POST /rooms/:room_id/redact/:event_id as power user redacts message -xxx POST /rooms/:room_id/redact/:event_id as original message sender redacts message -xxx POST /rooms/:room_id/redact/:event_id as random user does not redact message -xxx POST /redact disallows redaction of event in different room +xxx PUT /rooms/:room_id/redact/:event_id/:txn_id as power user redacts message +xxx PUT /rooms/:room_id/redact/:event_id/:txn_id as original message sender redacts message +xxx PUT /rooms/:room_id/redact/:event_id/:txn_id as random user does not redact message +xxx PUT /redact disallows redaction of event in different room xxx Redaction of a redaction redacts the redaction reason +xxx PUT /rooms/:room_id/redact/:event_id/:txn_id is idempotent v1s A departed room is still included in /initialSync (SPEC-216) v1s Can get rooms/{roomId}/initialSync for a departed room (SPEC-216) rst Can get rooms/{roomId}/state for a departed room (SPEC-216) @@ -921,3 +922,18 @@ msc We can't peek into rooms with invited history_visibility msc We can't peek into rooms with joined history_visibility msc Local users can peek by room alias msc Peeked rooms only turn up in the sync for the device who peeked them +ban 'ban' event respects room powerlevel (2 subtests) +inv Test that we can be reinvited to a room we created (11 subtests) +fiv Rejecting invite over federation doesn't break incremental /sync +pre Presence can be set from sync +fst /state returns M_NOT_FOUND for an outlier +fst /state_ids returns M_NOT_FOUND for an outlier +fst /state returns M_NOT_FOUND for a rejected message event +fst /state_ids returns M_NOT_FOUND for a rejected message event +fst /state returns M_NOT_FOUND for a rejected state event +fst /state_ids returns M_NOT_FOUND for a rejected state event +fst Room state after a rejected message event is the same as before +fst Room state after a rejected state event is the same as before +fpb Federation publicRoom Name/topic keys are correct +fed New federated private chats get full presence information (SYN-115) (10 subtests) +dvk Rejects invalid device keys \ No newline at end of file From c07f347f0054515a41834e171759569dc513187e Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Fri, 22 Apr 2022 11:38:29 +0200 Subject: [PATCH 06/50] Reuse the existing lazyload cache on /context and /messages (#2367) --- syncapi/routing/context.go | 23 ++++++++++++++++++----- syncapi/routing/messages.go | 9 ++++++++- syncapi/routing/routing.go | 5 ++++- syncapi/syncapi.go | 2 +- 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go index aaa0c61bf..17215b669 100644 --- a/syncapi/routing/context.go +++ b/syncapi/routing/context.go @@ -22,6 +22,7 @@ import ( "strconv" "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/internal/caching" roomserver "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -44,6 +45,7 @@ func Context( rsAPI roomserver.RoomserverInternalAPI, syncDB storage.Database, roomID, eventID string, + lazyLoadCache *caching.LazyLoadCache, ) util.JSONResponse { filter, err := parseRoomEventFilter(req) if err != nil { @@ -129,7 +131,7 @@ func Context( eventsBeforeClient := gomatrixserverlib.HeaderedToClientEvents(eventsBefore, gomatrixserverlib.FormatAll) eventsAfterClient := gomatrixserverlib.HeaderedToClientEvents(eventsAfter, gomatrixserverlib.FormatAll) - newState := applyLazyLoadMembers(filter, eventsAfterClient, eventsBeforeClient, state) + newState := applyLazyLoadMembers(device, filter, eventsAfterClient, eventsBeforeClient, state, lazyLoadCache) response := ContextRespsonse{ Event: gomatrixserverlib.HeaderedToClientEvent(&requestedEvent, gomatrixserverlib.FormatAll), @@ -148,15 +150,25 @@ func Context( } } -func applyLazyLoadMembers(filter *gomatrixserverlib.RoomEventFilter, eventsAfter, eventsBefore []gomatrixserverlib.ClientEvent, state []*gomatrixserverlib.HeaderedEvent) []*gomatrixserverlib.HeaderedEvent { +func applyLazyLoadMembers( + device *userapi.Device, + filter *gomatrixserverlib.RoomEventFilter, + eventsAfter, eventsBefore []gomatrixserverlib.ClientEvent, + state []*gomatrixserverlib.HeaderedEvent, + lazyLoadCache *caching.LazyLoadCache, +) []*gomatrixserverlib.HeaderedEvent { if filter == nil || !filter.LazyLoadMembers { return state } allEvents := append(eventsBefore, eventsAfter...) - x := make(map[string]bool) + x := make(map[string]struct{}) // get members who actually send an event for _, e := range allEvents { - x[e.Sender] = true + // Don't add membership events the client should already know about + if _, cached := lazyLoadCache.IsLazyLoadedUserCached(device, e.RoomID, e.Sender); cached { + continue + } + x[e.Sender] = struct{}{} } newState := []*gomatrixserverlib.HeaderedEvent{} @@ -166,8 +178,9 @@ func applyLazyLoadMembers(filter *gomatrixserverlib.RoomEventFilter, eventsAfter newState = append(newState, event) } else { // did the user send an event? - if x[event.Sender()] { + if _, ok := x[event.Sender()]; ok { membershipEvents = append(membershipEvents, event) + lazyLoadCache.StoreLazyLoadedUser(device, event.RoomID(), event.Sender(), event.EventID()) } } } diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 519aeff68..f34901bf2 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -21,6 +21,7 @@ import ( "sort" "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/syncapi/storage" @@ -64,6 +65,7 @@ func OnIncomingMessagesRequest( rsAPI api.RoomserverInternalAPI, cfg *config.SyncAPI, srp *sync.RequestPool, + lazyLoadCache *caching.LazyLoadCache, ) util.JSONResponse { var err error @@ -200,6 +202,10 @@ func OnIncomingMessagesRequest( if filter.LazyLoadMembers { membershipToUser := make(map[string]*gomatrixserverlib.HeaderedEvent) for _, evt := range clientEvents { + // Don't add membership events the client should already know about + if _, cached := lazyLoadCache.IsLazyLoadedUserCached(device, roomID, evt.Sender); cached { + continue + } membership, err := db.GetStateEvent(req.Context(), roomID, gomatrixserverlib.MRoomMember, evt.Sender) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("failed to get membership event for user") @@ -207,10 +213,11 @@ func OnIncomingMessagesRequest( } if membership != nil { membershipToUser[evt.Sender] = membership + lazyLoadCache.StoreLazyLoadedUser(device, roomID, evt.Sender, membership.EventID()) } } for _, evt := range membershipToUser { - state = append(state, gomatrixserverlib.HeaderedToClientEvent(evt, gomatrixserverlib.FormatAll)) + state = append(state, gomatrixserverlib.HeaderedToClientEvent(evt, gomatrixserverlib.FormatSync)) } } diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index be366ba10..4102cf073 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -18,6 +18,7 @@ import ( "net/http" "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" @@ -38,6 +39,7 @@ func Setup( userAPI userapi.UserInternalAPI, federation *gomatrixserverlib.FederationClient, rsAPI api.RoomserverInternalAPI, cfg *config.SyncAPI, + lazyLoadCache *caching.LazyLoadCache, ) { v3mux := csMux.PathPrefix("/{apiversion:(?:r0|v3)}/").Subrouter() @@ -51,7 +53,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], device, federation, rsAPI, cfg, srp) + return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], device, federation, rsAPI, cfg, srp, lazyLoadCache) })).Methods(http.MethodGet, http.MethodOptions) v3mux.Handle("/user/{userId}/filter", @@ -89,6 +91,7 @@ func Setup( req, device, rsAPI, syncDB, vars["roomId"], vars["eventId"], + lazyLoadCache, ) }), ).Methods(http.MethodGet, http.MethodOptions) diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index 2f9165d91..b2d333f74 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -148,5 +148,5 @@ func AddPublicRoutes( logrus.WithError(err).Panicf("failed to start presence consumer") } - routing.Setup(router, requestPool, syncDB, userAPI, federation, rsAPI, cfg) + routing.Setup(router, requestPool, syncDB, userAPI, federation, rsAPI, cfg, lazyLoadCache) } From 6d78c4d67d5096f5bb2c7cd56f15ab79949edc86 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 22 Apr 2022 14:58:24 +0100 Subject: [PATCH 07/50] Fix retrieving cross-signing signatures in `/user/devices/{userId}` (#2368) * Fix retrieving cross-signing signatures in `/user/devices/{userId}` We need to know the target device IDs in order to get the signatures and we weren't populating those. * Fix up signature retrieval * Fix SQLite * Always include the target's own signatures as well as the requesting user --- federationapi/routing/devices.go | 3 ++ keyserver/internal/cross_signing.go | 21 ++++++------- keyserver/internal/internal.go | 30 ++++++++++++++++--- keyserver/storage/interface.go | 2 +- .../postgres/cross_signing_sigs_table.go | 6 ++-- keyserver/storage/shared/storage.go | 6 ++-- .../sqlite3/cross_signing_sigs_table.go | 8 ++--- keyserver/storage/tables/interface.go | 2 +- 8 files changed, 52 insertions(+), 26 deletions(-) diff --git a/federationapi/routing/devices.go b/federationapi/routing/devices.go index 4cd199960..8890eac4b 100644 --- a/federationapi/routing/devices.go +++ b/federationapi/routing/devices.go @@ -43,6 +43,9 @@ func GetUserDevices( }, } sigRes := &keyapi.QuerySignaturesResponse{} + for _, dev := range res.Devices { + sigReq.TargetIDs[userID] = append(sigReq.TargetIDs[userID], gomatrixserverlib.KeyID(dev.DeviceID)) + } keyAPI.QuerySignatures(req.Context(), sigReq, sigRes) response := gomatrixserverlib.RespUserDevices{ diff --git a/keyserver/internal/cross_signing.go b/keyserver/internal/cross_signing.go index 0d083b4ba..2281f4bbf 100644 --- a/keyserver/internal/cross_signing.go +++ b/keyserver/internal/cross_signing.go @@ -455,10 +455,10 @@ func (a *KeyInternalAPI) processOtherSignatures( func (a *KeyInternalAPI) crossSigningKeysFromDatabase( ctx context.Context, req *api.QueryKeysRequest, res *api.QueryKeysResponse, ) { - for userID := range req.UserToDevices { - keys, err := a.DB.CrossSigningKeysForUser(ctx, userID) + for targetUserID := range req.UserToDevices { + keys, err := a.DB.CrossSigningKeysForUser(ctx, targetUserID) if err != nil { - logrus.WithError(err).Errorf("Failed to get cross-signing keys for user %q", userID) + logrus.WithError(err).Errorf("Failed to get cross-signing keys for user %q", targetUserID) continue } @@ -469,9 +469,9 @@ func (a *KeyInternalAPI) crossSigningKeysFromDatabase( break } - sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, userID, keyID) + sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, req.UserID, targetUserID, keyID) if err != nil && err != sql.ErrNoRows { - logrus.WithError(err).Errorf("Failed to get cross-signing signatures for user %q key %q", userID, keyID) + logrus.WithError(err).Errorf("Failed to get cross-signing signatures for user %q key %q", targetUserID, keyID) continue } @@ -491,7 +491,7 @@ func (a *KeyInternalAPI) crossSigningKeysFromDatabase( case req.UserID != "" && originUserID == req.UserID: // Include signatures that we created appendSignature(originUserID, originKeyID, signature) - case originUserID == userID: + case originUserID == targetUserID: // Include signatures that were created by the person whose key // we are processing appendSignature(originUserID, originKeyID, signature) @@ -501,13 +501,13 @@ func (a *KeyInternalAPI) crossSigningKeysFromDatabase( switch keyType { case gomatrixserverlib.CrossSigningKeyPurposeMaster: - res.MasterKeys[userID] = key + res.MasterKeys[targetUserID] = key case gomatrixserverlib.CrossSigningKeyPurposeSelfSigning: - res.SelfSigningKeys[userID] = key + res.SelfSigningKeys[targetUserID] = key case gomatrixserverlib.CrossSigningKeyPurposeUserSigning: - res.UserSigningKeys[userID] = key + res.UserSigningKeys[targetUserID] = key } } } @@ -546,7 +546,8 @@ func (a *KeyInternalAPI) QuerySignatures(ctx context.Context, req *api.QuerySign } for _, targetKeyID := range forTargetUser { - sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, targetUserID, targetKeyID) + // Get own signatures only. + sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, targetUserID, targetUserID, targetKeyID) if err != nil && err != sql.ErrNoRows { res.Error = &api.KeyError{ Err: fmt.Sprintf("a.DB.CrossSigningSigsForTarget: %s", err), diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index a05476f5f..e70de7671 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -313,9 +313,31 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques // Finally, append signatures that we know about // TODO: This is horrible because we need to round-trip the signature from // JSON, add the signatures and marshal it again, for some reason? - for userID, forUserID := range res.DeviceKeys { - for keyID, key := range forUserID { - sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, userID, gomatrixserverlib.KeyID(keyID)) + + for targetUserID, masterKey := range res.MasterKeys { + for targetKeyID := range masterKey.Keys { + sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, req.UserID, targetUserID, targetKeyID) + if err != nil { + logrus.WithError(err).Errorf("a.DB.CrossSigningSigsForTarget failed") + continue + } + if len(sigMap) == 0 { + continue + } + for sourceUserID, forSourceUser := range sigMap { + for sourceKeyID, sourceSig := range forSourceUser { + if _, ok := masterKey.Signatures[sourceUserID]; !ok { + masterKey.Signatures[sourceUserID] = map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{} + } + masterKey.Signatures[sourceUserID][sourceKeyID] = sourceSig + } + } + } + } + + for targetUserID, forUserID := range res.DeviceKeys { + for targetKeyID, key := range forUserID { + sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, req.UserID, targetUserID, gomatrixserverlib.KeyID(targetKeyID)) if err != nil { logrus.WithError(err).Errorf("a.DB.CrossSigningSigsForTarget failed") continue @@ -339,7 +361,7 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques } } if js, err := json.Marshal(deviceKey); err == nil { - res.DeviceKeys[userID][keyID] = js + res.DeviceKeys[targetUserID][targetKeyID] = js } } } diff --git a/keyserver/storage/interface.go b/keyserver/storage/interface.go index 16e034776..242e16a06 100644 --- a/keyserver/storage/interface.go +++ b/keyserver/storage/interface.go @@ -81,7 +81,7 @@ type Database interface { CrossSigningKeysForUser(ctx context.Context, userID string) (map[gomatrixserverlib.CrossSigningKeyPurpose]gomatrixserverlib.CrossSigningKey, error) CrossSigningKeysDataForUser(ctx context.Context, userID string) (types.CrossSigningKeyMap, error) - CrossSigningSigsForTarget(ctx context.Context, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (types.CrossSigningSigMap, error) + CrossSigningSigsForTarget(ctx context.Context, originUserID, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (types.CrossSigningSigMap, error) StoreCrossSigningKeysForUser(ctx context.Context, userID string, keyMap types.CrossSigningKeyMap) error StoreCrossSigningSigsForTarget(ctx context.Context, originUserID string, originKeyID gomatrixserverlib.KeyID, targetUserID string, targetKeyID gomatrixserverlib.KeyID, signature gomatrixserverlib.Base64Bytes) error diff --git a/keyserver/storage/postgres/cross_signing_sigs_table.go b/keyserver/storage/postgres/cross_signing_sigs_table.go index e11853957..40633c05c 100644 --- a/keyserver/storage/postgres/cross_signing_sigs_table.go +++ b/keyserver/storage/postgres/cross_signing_sigs_table.go @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS keyserver_cross_signing_sigs ( const selectCrossSigningSigsForTargetSQL = "" + "SELECT origin_user_id, origin_key_id, signature FROM keyserver_cross_signing_sigs" + - " WHERE target_user_id = $1 AND target_key_id = $2" + " WHERE (origin_user_id = $1 OR origin_user_id = target_user_id) AND target_user_id = $2 AND target_key_id = $3" const upsertCrossSigningSigsForTargetSQL = "" + "INSERT INTO keyserver_cross_signing_sigs (origin_user_id, origin_key_id, target_user_id, target_key_id, signature)" + @@ -72,9 +72,9 @@ func NewPostgresCrossSigningSigsTable(db *sql.DB) (tables.CrossSigningSigs, erro } func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget( - ctx context.Context, txn *sql.Tx, targetUserID string, targetKeyID gomatrixserverlib.KeyID, + ctx context.Context, txn *sql.Tx, originUserID, targetUserID string, targetKeyID gomatrixserverlib.KeyID, ) (r types.CrossSigningSigMap, err error) { - rows, err := sqlutil.TxStmt(txn, s.selectCrossSigningSigsForTargetStmt).QueryContext(ctx, targetUserID, targetKeyID) + rows, err := sqlutil.TxStmt(txn, s.selectCrossSigningSigsForTargetStmt).QueryContext(ctx, originUserID, targetUserID, targetKeyID) if err != nil { return nil, err } diff --git a/keyserver/storage/shared/storage.go b/keyserver/storage/shared/storage.go index 7ba0b3ea1..0e587b5a8 100644 --- a/keyserver/storage/shared/storage.go +++ b/keyserver/storage/shared/storage.go @@ -190,7 +190,7 @@ func (d *Database) CrossSigningKeysForUser(ctx context.Context, userID string) ( keyID: key, }, } - sigMap, err := d.CrossSigningSigsTable.SelectCrossSigningSigsForTarget(ctx, nil, userID, keyID) + sigMap, err := d.CrossSigningSigsTable.SelectCrossSigningSigsForTarget(ctx, nil, userID, userID, keyID) if err != nil { continue } @@ -219,8 +219,8 @@ func (d *Database) CrossSigningKeysDataForUser(ctx context.Context, userID strin } // CrossSigningSigsForTarget returns the signatures for a given user's key ID, if any. -func (d *Database) CrossSigningSigsForTarget(ctx context.Context, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (types.CrossSigningSigMap, error) { - return d.CrossSigningSigsTable.SelectCrossSigningSigsForTarget(ctx, nil, targetUserID, targetKeyID) +func (d *Database) CrossSigningSigsForTarget(ctx context.Context, originUserID, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (types.CrossSigningSigMap, error) { + return d.CrossSigningSigsTable.SelectCrossSigningSigsForTarget(ctx, nil, originUserID, targetUserID, targetKeyID) } // StoreCrossSigningKeysForUser stores the latest known cross-signing keys for a user. diff --git a/keyserver/storage/sqlite3/cross_signing_sigs_table.go b/keyserver/storage/sqlite3/cross_signing_sigs_table.go index 9abf54363..29ee889fd 100644 --- a/keyserver/storage/sqlite3/cross_signing_sigs_table.go +++ b/keyserver/storage/sqlite3/cross_signing_sigs_table.go @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS keyserver_cross_signing_sigs ( const selectCrossSigningSigsForTargetSQL = "" + "SELECT origin_user_id, origin_key_id, signature FROM keyserver_cross_signing_sigs" + - " WHERE target_user_id = $1 AND target_key_id = $2" + " WHERE (origin_user_id = $1 OR origin_user_id = target_user_id) AND target_user_id = $2 AND target_key_id = $3" const upsertCrossSigningSigsForTargetSQL = "" + "INSERT OR REPLACE INTO keyserver_cross_signing_sigs (origin_user_id, origin_key_id, target_user_id, target_key_id, signature)" + @@ -71,13 +71,13 @@ func NewSqliteCrossSigningSigsTable(db *sql.DB) (tables.CrossSigningSigs, error) } func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget( - ctx context.Context, txn *sql.Tx, targetUserID string, targetKeyID gomatrixserverlib.KeyID, + ctx context.Context, txn *sql.Tx, originUserID, targetUserID string, targetKeyID gomatrixserverlib.KeyID, ) (r types.CrossSigningSigMap, err error) { - rows, err := sqlutil.TxStmt(txn, s.selectCrossSigningSigsForTargetStmt).QueryContext(ctx, targetUserID, targetKeyID) + rows, err := sqlutil.TxStmt(txn, s.selectCrossSigningSigsForTargetStmt).QueryContext(ctx, originUserID, targetUserID, targetKeyID) if err != nil { return nil, err } - defer internal.CloseAndLogIfError(ctx, rows, "selectCrossSigningSigsForTargetStmt: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectCrossSigningSigsForOriginTargetStmt: rows.close() failed") r = types.CrossSigningSigMap{} for rows.Next() { var userID string diff --git a/keyserver/storage/tables/interface.go b/keyserver/storage/tables/interface.go index f840cd1f3..37a010a7c 100644 --- a/keyserver/storage/tables/interface.go +++ b/keyserver/storage/tables/interface.go @@ -64,7 +64,7 @@ type CrossSigningKeys interface { } type CrossSigningSigs interface { - SelectCrossSigningSigsForTarget(ctx context.Context, txn *sql.Tx, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (r types.CrossSigningSigMap, err error) + SelectCrossSigningSigsForTarget(ctx context.Context, txn *sql.Tx, originUserID, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (r types.CrossSigningSigMap, err error) UpsertCrossSigningSigsForTarget(ctx context.Context, txn *sql.Tx, originUserID string, originKeyID gomatrixserverlib.KeyID, targetUserID string, targetKeyID gomatrixserverlib.KeyID, signature gomatrixserverlib.Base64Bytes) error DeleteCrossSigningSigsForTarget(ctx context.Context, txn *sql.Tx, targetUserID string, targetKeyID gomatrixserverlib.KeyID) error } From 67fb086c13b6845e6a76ab89d314895306e14e96 Mon Sep 17 00:00:00 2001 From: Till Faelligen Date: Sun, 24 Apr 2022 20:26:20 +0200 Subject: [PATCH 08/50] Update README --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index cbb35ad59..75c827c33 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,11 @@ It intends to provide an **efficient**, **reliable** and **scalable** alternativ - Efficient: A small memory footprint with better baseline performance than an out-of-the-box Synapse. - Reliable: Implements the Matrix specification as written, using the - [same test suite](https://github.com/matrix-org/sytest) as Synapse as well as - a [brand new Go test suite](https://github.com/matrix-org/complement). + [same test suite](https://github.com/matrix-org/sytest) as Synapse as well as + a [brand new Go test suite](https://github.com/matrix-org/complement). - Scalable: can run on multiple machines and eventually scale to massive homeserver deployments. -As of October 2020, Dendrite has now entered **beta** which means: +As of October 2020 (current [progress below](#progress)), Dendrite has now entered **beta** which means: - Dendrite is ready for early adopters. We recommend running in Monolith mode with a PostgreSQL database. - Dendrite has periodic semver releases. We intend to release new versions as we land significant features. @@ -21,7 +21,7 @@ This does not mean: - Dendrite is bug-free. It has not yet been battle-tested in the real world and so will be error prone initially. - All of the CS/Federation APIs are implemented. We are tracking progress via a script called 'Are We Synapse Yet?'. In particular, - presence and push notifications are entirely missing from Dendrite. See [CHANGES.md](CHANGES.md) for updates. + presence and push notifications are entirely missing from Dendrite. See [CHANGES.md](CHANGES.md) for updates. - Dendrite is ready for massive homeserver deployments. You cannot shard each microservice, only run each one on a different machine. Currently, we expect Dendrite to function well for small (10s/100s of users) homeserver deployments as well as P2P Matrix nodes in-browser or on mobile devices. @@ -78,7 +78,7 @@ $ ./bin/dendrite-monolith-server --tls-cert server.crt --tls-key server.key --co Then point your favourite Matrix client at `http://localhost:8008` or `https://localhost:8448`. -## Progress +## Progress We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it From 446819e4ac405393ae7834107adc5761afce8a34 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Mon, 25 Apr 2022 11:56:50 +0200 Subject: [PATCH 09/50] Store the EDU type in the database (#2370) --- federationapi/queue/destinationqueue.go | 1 + federationapi/storage/interface.go | 2 +- federationapi/storage/shared/storage_edus.go | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/federationapi/queue/destinationqueue.go b/federationapi/queue/destinationqueue.go index 09814b31f..a5f8c03b9 100644 --- a/federationapi/queue/destinationqueue.go +++ b/federationapi/queue/destinationqueue.go @@ -125,6 +125,7 @@ func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, receipt *share context.TODO(), oq.destination, // the destination server name receipt, // NIDs from federationapi_queue_json table + event.Type, ); err != nil { logrus.WithError(err).Errorf("failed to associate EDU with destination %q", oq.destination) return diff --git a/federationapi/storage/interface.go b/federationapi/storage/interface.go index 3fa8d1f7a..e3038651b 100644 --- a/federationapi/storage/interface.go +++ b/federationapi/storage/interface.go @@ -39,7 +39,7 @@ type Database interface { GetPendingEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (edus map[*shared.Receipt]*gomatrixserverlib.EDU, err error) AssociatePDUWithDestination(ctx context.Context, transactionID gomatrixserverlib.TransactionID, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt) error - AssociateEDUWithDestination(ctx context.Context, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt) error + AssociateEDUWithDestination(ctx context.Context, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt, eduType string) error CleanPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error CleanEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error diff --git a/federationapi/storage/shared/storage_edus.go b/federationapi/storage/shared/storage_edus.go index 6e3c7e367..02a23338f 100644 --- a/federationapi/storage/shared/storage_edus.go +++ b/federationapi/storage/shared/storage_edus.go @@ -31,12 +31,13 @@ func (d *Database) AssociateEDUWithDestination( ctx context.Context, serverName gomatrixserverlib.ServerName, receipt *Receipt, + eduType string, ) error { return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { if err := d.FederationQueueEDUs.InsertQueueEDU( ctx, // context txn, // SQL transaction - "", // TODO: EDU type for coalescing + eduType, // EDU type for coalescing serverName, // destination server name receipt.nid, // NID from the federationapi_queue_json table ); err != nil { From aad81b7b4dcf971508cde266c5ae99e35261bf27 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 25 Apr 2022 14:22:46 +0100 Subject: [PATCH 10/50] Only call key update process functions if there are updates, don't send things to ourselves over federation --- federationapi/queue/queue.go | 2 ++ federationapi/routing/send.go | 49 +++++++++++++++++++++++++++++----- keyserver/internal/internal.go | 9 +++++-- 3 files changed, 51 insertions(+), 9 deletions(-) diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go index 5b5481274..c45bbd1d4 100644 --- a/federationapi/queue/queue.go +++ b/federationapi/queue/queue.go @@ -210,6 +210,7 @@ func (oqs *OutgoingQueues) SendEvent( destmap[d] = struct{}{} } delete(destmap, oqs.origin) + delete(destmap, oqs.signing.ServerName) // Check if any of the destinations are prohibited by server ACLs. for destination := range destmap { @@ -275,6 +276,7 @@ func (oqs *OutgoingQueues) SendEDU( destmap[d] = struct{}{} } delete(destmap, oqs.origin) + delete(destmap, oqs.signing.ServerName) // There is absolutely no guarantee that the EDU will have a room_id // field, as it is not required by the spec. However, if it *does* diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index f2b902b6f..2c01afb1b 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -124,6 +124,7 @@ func Send( t := txnReq{ rsAPI: rsAPI, keys: keys, + ourServerName: cfg.Matrix.ServerName, federation: federation, servers: servers, keyAPI: keyAPI, @@ -183,6 +184,7 @@ type txnReq struct { gomatrixserverlib.Transaction rsAPI api.RoomserverInternalAPI keyAPI keyapi.KeyInternalAPI + ourServerName gomatrixserverlib.ServerName keys gomatrixserverlib.JSONVerifier federation txnFederationClient roomsMu *internal.MutexByRoom @@ -303,6 +305,7 @@ func (t *txnReq) processTransaction(ctx context.Context) (*gomatrixserverlib.Res return &gomatrixserverlib.RespSend{PDUs: results}, nil } +// nolint:gocyclo func (t *txnReq) processEDUs(ctx context.Context) { for _, e := range t.EDUs { eduCountTotal.Inc() @@ -318,13 +321,11 @@ func (t *txnReq) processEDUs(ctx context.Context) { util.GetLogger(ctx).WithError(err).Debug("Failed to unmarshal typing event") continue } - _, domain, err := gomatrixserverlib.SplitID('@', typingPayload.UserID) - if err != nil { - util.GetLogger(ctx).WithError(err).Debug("Failed to split domain from typing event sender") + if _, serverName, err := gomatrixserverlib.SplitID('@', typingPayload.UserID); err != nil { continue - } - if domain != t.Origin { - util.GetLogger(ctx).Debugf("Dropping typing event where sender domain (%q) doesn't match origin (%q)", domain, t.Origin) + } else if serverName == t.ourServerName { + continue + } else if serverName != t.Origin { continue } if err := t.producer.SendTyping(ctx, typingPayload.UserID, typingPayload.RoomID, typingPayload.Typing, 30*1000); err != nil { @@ -337,6 +338,13 @@ func (t *txnReq) processEDUs(ctx context.Context) { util.GetLogger(ctx).WithError(err).Debug("Failed to unmarshal send-to-device events") continue } + if _, serverName, err := gomatrixserverlib.SplitID('@', directPayload.Sender); err != nil { + continue + } else if serverName == t.ourServerName { + continue + } else if serverName != t.Origin { + continue + } for userID, byUser := range directPayload.Messages { for deviceID, message := range byUser { // TODO: check that the user and the device actually exist here @@ -405,6 +413,13 @@ func (t *txnReq) processPresence(ctx context.Context, e gomatrixserverlib.EDU) e return err } for _, content := range payload.Push { + if _, serverName, err := gomatrixserverlib.SplitID('@', content.UserID); err != nil { + continue + } else if serverName == t.ourServerName { + continue + } else if serverName != t.Origin { + continue + } presence, ok := syncTypes.PresenceFromString(content.Presence) if !ok { continue @@ -424,7 +439,13 @@ func (t *txnReq) processSigningKeyUpdate(ctx context.Context, e gomatrixserverli }).Debug("Failed to unmarshal signing key update") return err } - + if _, serverName, err := gomatrixserverlib.SplitID('@', updatePayload.UserID); err != nil { + return nil + } else if serverName == t.ourServerName { + return nil + } else if serverName != t.Origin { + return nil + } keys := gomatrixserverlib.CrossSigningKeys{} if updatePayload.MasterKey != nil { keys.MasterKey = *updatePayload.MasterKey @@ -450,6 +471,13 @@ func (t *txnReq) processReceiptEvent(ctx context.Context, timestamp gomatrixserverlib.Timestamp, eventIDs []string, ) error { + if _, serverName, err := gomatrixserverlib.SplitID('@', userID); err != nil { + return nil + } else if serverName == t.ourServerName { + return nil + } else if serverName != t.Origin { + return nil + } // store every event for _, eventID := range eventIDs { if err := t.producer.SendReceipt(ctx, userID, roomID, eventID, receiptType, timestamp); err != nil { @@ -466,6 +494,13 @@ func (t *txnReq) processDeviceListUpdate(ctx context.Context, e gomatrixserverli util.GetLogger(ctx).WithError(err).Error("Failed to unmarshal device list update event") return } + if _, serverName, err := gomatrixserverlib.SplitID('@', payload.UserID); err != nil { + return + } else if serverName == t.ourServerName { + return + } else if serverName != t.Origin { + return + } var inputRes keyapi.InputDeviceListUpdateResponse t.keyAPI.InputDeviceListUpdate(context.Background(), &keyapi.InputDeviceListUpdateRequest{ Event: payload, diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index e70de7671..e571c7e56 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -71,8 +71,12 @@ func (a *KeyInternalAPI) QueryKeyChanges(ctx context.Context, req *api.QueryKeyC func (a *KeyInternalAPI) PerformUploadKeys(ctx context.Context, req *api.PerformUploadKeysRequest, res *api.PerformUploadKeysResponse) { res.KeyErrors = make(map[string]map[string]*api.KeyError) - a.uploadLocalDeviceKeys(ctx, req, res) - a.uploadOneTimeKeys(ctx, req, res) + if len(req.DeviceKeys) > 0 { + a.uploadLocalDeviceKeys(ctx, req, res) + } + if len(req.OneTimeKeys) > 0 { + a.uploadOneTimeKeys(ctx, req, res) + } } func (a *KeyInternalAPI) PerformClaimKeys(ctx context.Context, req *api.PerformClaimKeysRequest, res *api.PerformClaimKeysResponse) { @@ -663,6 +667,7 @@ func (a *KeyInternalAPI) uploadLocalDeviceKeys(ctx context.Context, req *api.Per // add the display name field from keysToStore into existingKeys keysToStore = appendDisplayNames(existingKeys, keysToStore) } + // store the device keys and emit changes err = a.DB.StoreLocalDeviceKeys(ctx, keysToStore) if err != nil { From e95fc5c5e3e4945949fbc1e9036c599687925a4d Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Mon, 25 Apr 2022 19:04:46 +0200 Subject: [PATCH 11/50] Use provided filter for account_data (#2372) * Reuse IncrementalSync, use provided filter * Inform SyncAPI about newly created push_rules --- syncapi/streams/stream_accountdata.go | 35 ++------------------------- sytest-blacklist | 4 --- sytest-whitelist | 2 +- userapi/internal/api.go | 7 ++++++ 4 files changed, 10 insertions(+), 38 deletions(-) diff --git a/syncapi/streams/stream_accountdata.go b/syncapi/streams/stream_accountdata.go index 105d85260..094c51485 100644 --- a/syncapi/streams/stream_accountdata.go +++ b/syncapi/streams/stream_accountdata.go @@ -30,37 +30,7 @@ func (p *AccountDataStreamProvider) CompleteSync( ctx context.Context, req *types.SyncRequest, ) types.StreamPosition { - dataReq := &userapi.QueryAccountDataRequest{ - UserID: req.Device.UserID, - } - dataRes := &userapi.QueryAccountDataResponse{} - if err := p.userAPI.QueryAccountData(ctx, dataReq, dataRes); err != nil { - req.Log.WithError(err).Error("p.userAPI.QueryAccountData failed") - return p.LatestPosition(ctx) - } - for datatype, databody := range dataRes.GlobalAccountData { - req.Response.AccountData.Events = append( - req.Response.AccountData.Events, - gomatrixserverlib.ClientEvent{ - Type: datatype, - Content: gomatrixserverlib.RawJSON(databody), - }, - ) - } - for r, j := range req.Response.Rooms.Join { - for datatype, databody := range dataRes.RoomAccountData[r] { - j.AccountData.Events = append( - j.AccountData.Events, - gomatrixserverlib.ClientEvent{ - Type: datatype, - Content: gomatrixserverlib.RawJSON(databody), - }, - ) - req.Response.Rooms.Join[r] = j - } - } - - return p.LatestPosition(ctx) + return p.IncrementalSync(ctx, req, 0, p.LatestPosition(ctx)) } func (p *AccountDataStreamProvider) IncrementalSync( @@ -72,10 +42,9 @@ func (p *AccountDataStreamProvider) IncrementalSync( From: from, To: to, } - accountDataFilter := gomatrixserverlib.DefaultEventFilter() // TODO: use filter provided in req instead dataTypes, err := p.DB.GetAccountDataInRange( - ctx, req.Device.UserID, r, &accountDataFilter, + ctx, req.Device.UserID, r, &req.Filter.AccountData, ) if err != nil { req.Log.WithError(err).Error("p.DB.GetAccountDataInRange failed") diff --git a/sytest-blacklist b/sytest-blacklist index f1bd60db1..713a5b631 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -1,7 +1,3 @@ -# Blacklisted until matrix-org/dendrite#862 is reverted due to Riot bug - -Latest account data appears in v2 /sync - # Relies on a rejected PL event which will never be accepted into the DAG # Caused by diff --git a/sytest-whitelist b/sytest-whitelist index 979f12bf6..2052185fe 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -154,7 +154,7 @@ Can add account data Can add account data to room Can get account data without syncing Can get room account data without syncing -#Latest account data appears in v2 /sync +Latest account data appears in v2 /sync New account data appears in incremental v2 /sync Checking local federation server Inbound federation can query profile data diff --git a/userapi/internal/api.go b/userapi/internal/api.go index d1c12f05f..be58e2d8d 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -90,6 +90,13 @@ func (a *UserInternalAPI) PerformAccountCreation(ctx context.Context, req *api.P return nil } + // Inform the SyncAPI about the newly created push_rules + if err = a.SyncProducer.SendAccountData(acc.UserID, "", "m.push_rules"); err != nil { + util.GetLogger(ctx).WithFields(logrus.Fields{ + "user_id": acc.UserID, + }).WithError(err).Warn("failed to send account data to the SyncAPI") + } + if req.AccountType == api.AccountTypeGuest { res.AccountCreated = true res.Account = acc From e8ab2154aa57d46bffa2db1e7c1333c2fa40cb21 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Mon, 25 Apr 2022 19:05:01 +0200 Subject: [PATCH 12/50] Return M_NOT_FOUND for rejected events (#2371) * Return M_NOT_FOUND for rejected events * Add passing tests --- federationapi/routing/state.go | 7 +++++++ roomserver/api/query.go | 2 ++ roomserver/internal/query/query.go | 24 +++++++++++++++++------- sytest-whitelist | 6 +++++- 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go index 37cbb9d1e..a202c92c2 100644 --- a/federationapi/routing/state.go +++ b/federationapi/routing/state.go @@ -129,6 +129,13 @@ func getState( return nil, nil, &resErr } + if response.IsRejected { + return nil, nil, &util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("Event not found"), + } + } + if !response.RoomExists { return nil, nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil} } diff --git a/roomserver/api/query.go b/roomserver/api/query.go index 8f84edcb5..ef2e6bb57 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -230,6 +230,8 @@ type QueryStateAndAuthChainResponse struct { // The lists will be in an arbitrary order. StateEvents []*gomatrixserverlib.HeaderedEvent `json:"state_events"` AuthChainEvents []*gomatrixserverlib.HeaderedEvent `json:"auth_chain_events"` + // True if the queried event was rejected earlier. + IsRejected bool `json:"is_rejected"` } // QueryRoomVersionCapabilitiesRequest asks for the default room version diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index 7e4d56684..5b33ec3c3 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -441,11 +441,11 @@ func (r *Queryer) QueryStateAndAuthChain( } var stateEvents []*gomatrixserverlib.Event - stateEvents, err = r.loadStateAtEventIDs(ctx, info, request.PrevEventIDs) + stateEvents, rejected, err := r.loadStateAtEventIDs(ctx, info, request.PrevEventIDs) if err != nil { return err } - + response.IsRejected = rejected response.PrevEventsExist = true // add the auth event IDs for the current state events too @@ -480,15 +480,23 @@ func (r *Queryer) QueryStateAndAuthChain( return err } -func (r *Queryer) loadStateAtEventIDs(ctx context.Context, roomInfo *types.RoomInfo, eventIDs []string) ([]*gomatrixserverlib.Event, error) { +func (r *Queryer) loadStateAtEventIDs(ctx context.Context, roomInfo *types.RoomInfo, eventIDs []string) ([]*gomatrixserverlib.Event, bool, error) { roomState := state.NewStateResolution(r.DB, roomInfo) prevStates, err := r.DB.StateAtEventIDs(ctx, eventIDs) if err != nil { switch err.(type) { case types.MissingEventError: - return nil, nil + return nil, false, nil default: - return nil, err + return nil, false, err + } + } + // Currently only used on /state and /state_ids + rejected := false + for i := range prevStates { + if prevStates[i].IsRejected { + rejected = true + break } } @@ -497,10 +505,12 @@ func (r *Queryer) loadStateAtEventIDs(ctx context.Context, roomInfo *types.RoomI ctx, prevStates, ) if err != nil { - return nil, err + return nil, rejected, err } - return helpers.LoadStateEvents(ctx, r.DB, stateEntries) + events, err := helpers.LoadStateEvents(ctx, r.DB, stateEntries) + + return events, rejected, err } type eventsFromIDs func(context.Context, []string) ([]types.Event, error) diff --git a/sytest-whitelist b/sytest-whitelist index 2052185fe..5d67aee6c 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -709,4 +709,8 @@ Gapped incremental syncs include all state changes Old leaves are present in gapped incremental syncs Leaves are present in non-gapped incremental syncs Members from the gap are included in gappy incr LL sync -Presence can be set from sync \ No newline at end of file +Presence can be set from sync +/state returns M_NOT_FOUND for a rejected message event +/state_ids returns M_NOT_FOUND for a rejected message event +/state returns M_NOT_FOUND for a rejected state event +/state_ids returns M_NOT_FOUND for a rejected state event \ No newline at end of file From 7df5d69a5b606e226c5d83c23aa9dd90785c0b2d Mon Sep 17 00:00:00 2001 From: Till Faelligen Date: Tue, 26 Apr 2022 08:07:27 +0200 Subject: [PATCH 13/50] Checkout correct branch for Sytest --- .github/workflows/dendrite.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index 4f337a866..5d60301c7 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -250,6 +250,7 @@ jobs: env: POSTGRES: ${{ matrix.postgres && 1}} API: ${{ matrix.api && 1 }} + SYTEST_BRANCH: ${{ github.head_ref }} steps: - uses: actions/checkout@v2 - name: Run Sytest From feac9db43fc459f1efa10424dfc96f8d54b55c64 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Tue, 26 Apr 2022 10:28:41 +0200 Subject: [PATCH 14/50] Add transactionsCache to redact endpoint (#2375) --- clientapi/routing/redaction.go | 20 +++++++++++++++++++- clientapi/routing/routing.go | 5 +++-- sytest-whitelist | 3 ++- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/clientapi/routing/redaction.go b/clientapi/routing/redaction.go index 01ea818ab..e8d14ce34 100644 --- a/clientapi/routing/redaction.go +++ b/clientapi/routing/redaction.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/internal/eventutil" + "github.com/matrix-org/dendrite/internal/transactions" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -40,12 +41,21 @@ type redactionResponse struct { func SendRedaction( req *http.Request, device *userapi.Device, roomID, eventID string, cfg *config.ClientAPI, rsAPI roomserverAPI.RoomserverInternalAPI, + txnID *string, + txnCache *transactions.Cache, ) util.JSONResponse { resErr := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID) if resErr != nil { return *resErr } + if txnID != nil { + // Try to fetch response from transactionsCache + if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID); ok { + return *res + } + } + ev := roomserverAPI.GetEvent(req.Context(), rsAPI, eventID) if ev == nil { return util.JSONResponse{ @@ -124,10 +134,18 @@ func SendRedaction( util.GetLogger(req.Context()).WithError(err).Errorf("failed to SendEvents") return jsonerror.InternalServerError() } - return util.JSONResponse{ + + res := util.JSONResponse{ Code: 200, JSON: redactionResponse{ EventID: e.EventID(), }, } + + // Add response to transactionsCache + if txnID != nil { + txnCache.AddTransaction(device.AccessToken, *txnID, &res) + } + + return res } diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 37d825b80..f370b4f8c 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -479,7 +479,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SendRedaction(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI) + return SendRedaction(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI, nil, nil) }), ).Methods(http.MethodPost, http.MethodOptions) v3mux.Handle("/rooms/{roomID}/redact/{eventID}/{txnId}", @@ -488,7 +488,8 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SendRedaction(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI) + txnID := vars["txnId"] + return SendRedaction(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI, &txnID, transactionsCache) }), ).Methods(http.MethodPut, http.MethodOptions) diff --git a/sytest-whitelist b/sytest-whitelist index 5d67aee6c..91304bd71 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -713,4 +713,5 @@ Presence can be set from sync /state returns M_NOT_FOUND for a rejected message event /state_ids returns M_NOT_FOUND for a rejected message event /state returns M_NOT_FOUND for a rejected state event -/state_ids returns M_NOT_FOUND for a rejected state event \ No newline at end of file +/state_ids returns M_NOT_FOUND for a rejected state event +PUT /rooms/:room_id/redact/:event_id/:txn_id is idempotent \ No newline at end of file From e8be2b234f616c8422372665c845d9a7a1af245f Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Tue, 26 Apr 2022 10:53:17 +0200 Subject: [PATCH 15/50] Add heroes to the room summary (#2373) * Implement room summary heroes * Add passing tests * Move MembershipCount to addRoomSummary * Add comments, close Statement --- syncapi/storage/interface.go | 1 + syncapi/storage/postgres/memberships_table.go | 38 ++++++++++--- syncapi/storage/shared/syncserver.go | 4 ++ syncapi/storage/sqlite3/memberships_table.go | 51 ++++++++++++++--- syncapi/storage/tables/interface.go | 1 + syncapi/streams/stream_pdu.go | 56 +++++++++++++++---- syncapi/streams/streams.go | 1 + sytest-whitelist | 5 +- 8 files changed, 131 insertions(+), 26 deletions(-) diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 14cb08a52..0fea88da6 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -39,6 +39,7 @@ type Database interface { GetStateDeltas(ctx context.Context, device *userapi.Device, r types.Range, userID string, stateFilter *gomatrixserverlib.StateFilter) ([]types.StateDelta, []string, error) RoomIDsWithMembership(ctx context.Context, userID string, membership string) ([]string, error) MembershipCount(ctx context.Context, roomID, membership string, pos types.StreamPosition) (int, error) + GetRoomHeroes(ctx context.Context, roomID, userID string, memberships []string) ([]string, error) RecentEvents(ctx context.Context, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error) diff --git a/syncapi/storage/postgres/memberships_table.go b/syncapi/storage/postgres/memberships_table.go index 39fa656cb..8c049977f 100644 --- a/syncapi/storage/postgres/memberships_table.go +++ b/syncapi/storage/postgres/memberships_table.go @@ -19,6 +19,8 @@ import ( "database/sql" "fmt" + "github.com/lib/pq" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" @@ -61,9 +63,13 @@ const selectMembershipCountSQL = "" + " SELECT DISTINCT ON (room_id, user_id) room_id, user_id, membership FROM syncapi_memberships WHERE room_id = $1 AND stream_pos <= $2 ORDER BY room_id, user_id, stream_pos DESC" + ") t WHERE t.membership = $3" +const selectHeroesSQL = "" + + "SELECT user_id FROM syncapi_memberships WHERE room_id = $1 AND user_id != $2 AND membership = ANY($3) LIMIT 5" + type membershipsStatements struct { upsertMembershipStmt *sql.Stmt selectMembershipCountStmt *sql.Stmt + selectHeroesStmt *sql.Stmt } func NewPostgresMembershipsTable(db *sql.DB) (tables.Memberships, error) { @@ -72,13 +78,11 @@ func NewPostgresMembershipsTable(db *sql.DB) (tables.Memberships, error) { if err != nil { return nil, err } - if s.upsertMembershipStmt, err = db.Prepare(upsertMembershipSQL); err != nil { - return nil, err - } - if s.selectMembershipCountStmt, err = db.Prepare(selectMembershipCountSQL); err != nil { - return nil, err - } - return s, nil + return s, sqlutil.StatementList{ + {&s.upsertMembershipStmt, upsertMembershipSQL}, + {&s.selectMembershipCountStmt, selectMembershipCountSQL}, + {&s.selectHeroesStmt, selectHeroesSQL}, + }.Prepare(db) } func (s *membershipsStatements) UpsertMembership( @@ -108,3 +112,23 @@ func (s *membershipsStatements) SelectMembershipCount( err = stmt.QueryRowContext(ctx, roomID, pos, membership).Scan(&count) return } + +func (s *membershipsStatements) SelectHeroes( + ctx context.Context, txn *sql.Tx, roomID, userID string, memberships []string, +) (heroes []string, err error) { + stmt := sqlutil.TxStmt(txn, s.selectHeroesStmt) + var rows *sql.Rows + rows, err = stmt.QueryContext(ctx, roomID, userID, pq.StringArray(memberships)) + if err != nil { + return + } + defer internal.CloseAndLogIfError(ctx, rows, "SelectHeroes: rows.close() failed") + var hero string + for rows.Next() { + if err = rows.Scan(&hero); err != nil { + return + } + heroes = append(heroes, hero) + } + return heroes, rows.Err() +} diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index 2143fd672..3c431db48 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -124,6 +124,10 @@ func (d *Database) MembershipCount(ctx context.Context, roomID, membership strin return d.Memberships.SelectMembershipCount(ctx, nil, roomID, membership, pos) } +func (d *Database) GetRoomHeroes(ctx context.Context, roomID, userID string, memberships []string) ([]string, error) { + return d.Memberships.SelectHeroes(ctx, nil, roomID, userID, memberships) +} + func (d *Database) RecentEvents(ctx context.Context, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error) { return d.OutputEvents.SelectRecentEvents(ctx, nil, roomID, r, eventFilter, chronologicalOrder, onlySyncEvents) } diff --git a/syncapi/storage/sqlite3/memberships_table.go b/syncapi/storage/sqlite3/memberships_table.go index 9f3530ccd..e4daa99c1 100644 --- a/syncapi/storage/sqlite3/memberships_table.go +++ b/syncapi/storage/sqlite3/memberships_table.go @@ -18,7 +18,9 @@ import ( "context" "database/sql" "fmt" + "strings" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" @@ -61,10 +63,14 @@ const selectMembershipCountSQL = "" + " SELECT * FROM syncapi_memberships WHERE room_id = $1 AND stream_pos <= $2 GROUP BY user_id HAVING(max(stream_pos))" + ") t WHERE t.membership = $3" +const selectHeroesSQL = "" + + "SELECT DISTINCT user_id FROM syncapi_memberships WHERE room_id = $1 AND user_id != $2 AND membership IN ($3) LIMIT 5" + type membershipsStatements struct { db *sql.DB upsertMembershipStmt *sql.Stmt selectMembershipCountStmt *sql.Stmt + //selectHeroesStmt *sql.Stmt - prepared at runtime due to variadic } func NewSqliteMembershipsTable(db *sql.DB) (tables.Memberships, error) { @@ -75,13 +81,11 @@ func NewSqliteMembershipsTable(db *sql.DB) (tables.Memberships, error) { if err != nil { return nil, err } - if s.upsertMembershipStmt, err = db.Prepare(upsertMembershipSQL); err != nil { - return nil, err - } - if s.selectMembershipCountStmt, err = db.Prepare(selectMembershipCountSQL); err != nil { - return nil, err - } - return s, nil + return s, sqlutil.StatementList{ + {&s.upsertMembershipStmt, upsertMembershipSQL}, + {&s.selectMembershipCountStmt, selectMembershipCountSQL}, + // {&s.selectHeroesStmt, selectHeroesSQL}, - prepared at runtime due to variadic + }.Prepare(db) } func (s *membershipsStatements) UpsertMembership( @@ -111,3 +115,36 @@ func (s *membershipsStatements) SelectMembershipCount( err = stmt.QueryRowContext(ctx, roomID, pos, membership).Scan(&count) return } + +func (s *membershipsStatements) SelectHeroes( + ctx context.Context, txn *sql.Tx, roomID, userID string, memberships []string, +) (heroes []string, err error) { + stmtSQL := strings.Replace(selectHeroesSQL, "($3)", sqlutil.QueryVariadicOffset(len(memberships), 2), 1) + stmt, err := s.db.PrepareContext(ctx, stmtSQL) + if err != nil { + return + } + defer internal.CloseAndLogIfError(ctx, stmt, "SelectHeroes: stmt.close() failed") + params := []interface{}{ + roomID, userID, + } + for _, membership := range memberships { + params = append(params, membership) + } + + stmt = sqlutil.TxStmt(txn, stmt) + var rows *sql.Rows + rows, err = stmt.QueryContext(ctx, params...) + if err != nil { + return + } + defer internal.CloseAndLogIfError(ctx, rows, "SelectHeroes: rows.close() failed") + var hero string + for rows.Next() { + if err = rows.Scan(&hero); err != nil { + return + } + heroes = append(heroes, hero) + } + return heroes, rows.Err() +} diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index 993e2022b..ac713dd5c 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -170,6 +170,7 @@ type Receipts interface { type Memberships interface { UpsertMembership(ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, streamPos, topologicalPos types.StreamPosition) error SelectMembershipCount(ctx context.Context, txn *sql.Tx, roomID, membership string, pos types.StreamPosition) (count int, err error) + SelectHeroes(ctx context.Context, txn *sql.Tx, roomID, userID string, memberships []string) (heroes []string, err error) } type NotificationData interface { diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index df5fb8e08..0d033095d 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -4,13 +4,16 @@ import ( "context" "database/sql" "fmt" + "sort" "sync" "time" "github.com/matrix-org/dendrite/internal/caching" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/types" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/tidwall/gjson" "go.uber.org/atomic" ) @@ -30,6 +33,7 @@ type PDUStreamProvider struct { workers atomic.Int32 // userID+deviceID -> lazy loading cache lazyLoadCache *caching.LazyLoadCache + rsAPI roomserverAPI.RoomserverInternalAPI } func (p *PDUStreamProvider) worker() { @@ -290,16 +294,11 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( } } - // Work out how many members are in the room. - joinedCount, _ := p.DB.MembershipCount(ctx, delta.RoomID, gomatrixserverlib.Join, latestPosition) - invitedCount, _ := p.DB.MembershipCount(ctx, delta.RoomID, gomatrixserverlib.Invite, latestPosition) - switch delta.Membership { case gomatrixserverlib.Join: jr := types.NewJoinResponse() if hasMembershipChange { - jr.Summary.JoinedMemberCount = &joinedCount - jr.Summary.InvitedMemberCount = &invitedCount + p.addRoomSummary(ctx, jr, delta.RoomID, device.UserID, latestPosition) } jr.Timeline.PrevBatch = &prevBatch jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) @@ -332,6 +331,45 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( return latestPosition, nil } +func (p *PDUStreamProvider) addRoomSummary(ctx context.Context, jr *types.JoinResponse, roomID, userID string, latestPosition types.StreamPosition) { + // Work out how many members are in the room. + joinedCount, _ := p.DB.MembershipCount(ctx, roomID, gomatrixserverlib.Join, latestPosition) + invitedCount, _ := p.DB.MembershipCount(ctx, roomID, gomatrixserverlib.Invite, latestPosition) + + jr.Summary.JoinedMemberCount = &joinedCount + jr.Summary.InvitedMemberCount = &invitedCount + + fetchStates := []gomatrixserverlib.StateKeyTuple{ + {EventType: gomatrixserverlib.MRoomName}, + {EventType: gomatrixserverlib.MRoomCanonicalAlias}, + } + // Check if the room has a name or a canonical alias + latestState := &roomserverAPI.QueryLatestEventsAndStateResponse{} + err := p.rsAPI.QueryLatestEventsAndState(ctx, &roomserverAPI.QueryLatestEventsAndStateRequest{StateToFetch: fetchStates, RoomID: roomID}, latestState) + if err != nil { + return + } + // Check if the room has a name or canonical alias, if so, return. + for _, ev := range latestState.StateEvents { + switch ev.Type() { + case gomatrixserverlib.MRoomName: + if gjson.GetBytes(ev.Content(), "name").Str != "" { + return + } + case gomatrixserverlib.MRoomCanonicalAlias: + if gjson.GetBytes(ev.Content(), "alias").Str != "" { + return + } + } + } + heroes, err := p.DB.GetRoomHeroes(ctx, roomID, userID, []string{"join", "invite"}) + if err != nil { + return + } + sort.Strings(heroes) + jr.Summary.Heroes = heroes +} + func (p *PDUStreamProvider) getJoinResponseForCompleteSync( ctx context.Context, roomID string, @@ -416,9 +454,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( prevBatch.Decrement() } - // Work out how many members are in the room. - joinedCount, _ := p.DB.MembershipCount(ctx, roomID, gomatrixserverlib.Join, r.From) - invitedCount, _ := p.DB.MembershipCount(ctx, roomID, gomatrixserverlib.Invite, r.From) + p.addRoomSummary(ctx, jr, roomID, device.UserID, r.From) // We don't include a device here as we don't need to send down // transaction IDs for complete syncs, but we do it anyway because Sytest demands it for: @@ -439,8 +475,6 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( } } - jr.Summary.JoinedMemberCount = &joinedCount - jr.Summary.InvitedMemberCount = &invitedCount jr.Timeline.PrevBatch = prevBatch jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = limited diff --git a/syncapi/streams/streams.go b/syncapi/streams/streams.go index d3195b78f..a18a0cc41 100644 --- a/syncapi/streams/streams.go +++ b/syncapi/streams/streams.go @@ -33,6 +33,7 @@ func NewSyncStreamProviders( PDUStreamProvider: &PDUStreamProvider{ StreamProvider: StreamProvider{DB: d}, lazyLoadCache: lazyLoadCache, + rsAPI: rsAPI, }, TypingStreamProvider: &TypingStreamProvider{ StreamProvider: StreamProvider{DB: d}, diff --git a/sytest-whitelist b/sytest-whitelist index 91304bd71..c9829606f 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -714,4 +714,7 @@ Presence can be set from sync /state_ids returns M_NOT_FOUND for a rejected message event /state returns M_NOT_FOUND for a rejected state event /state_ids returns M_NOT_FOUND for a rejected state event -PUT /rooms/:room_id/redact/:event_id/:txn_id is idempotent \ No newline at end of file +PUT /rooms/:room_id/redact/:event_id/:txn_id is idempotent +Unnamed room comes with a name summary +Named room comes with just joined member count summary +Room summary only has 5 heroes \ No newline at end of file From 5306c73b008567d855ca548d195abf3dfaf8917c Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 26 Apr 2022 13:08:54 +0100 Subject: [PATCH 16/50] Fix bug when uploading device signatures (#2377) * Find the complete key ID when uploading signatures * Try that again * Try splitting the right thing * Don't do it for device keys * Refactor `QuerySignatures` * Revert "Refactor `QuerySignatures`" This reverts commit c02832a3e92569f64f180dec1555056dc8f8c3e3. * Both requested key IDs and master/self/user keys * Fix uniqueness * Try tweaking GMSL * Update GMSL again * Revert "Update GMSL again" This reverts commit bd6916cc379dd8d9e3f38d979c6550bd658938aa. * Revert "Try tweaking GMSL" This reverts commit 2a054524da9d64c6a2a5228262fbba5fde28798c. * Database migrations --- keyserver/internal/cross_signing.go | 7 ++ .../postgres/cross_signing_sigs_table.go | 6 +- .../deltas/2022042612000000_xsigning_idx.go | 52 +++++++++++++ keyserver/storage/postgres/storage.go | 1 + .../sqlite3/cross_signing_sigs_table.go | 4 +- .../deltas/2022042612000000_xsigning_idx.go | 76 +++++++++++++++++++ keyserver/storage/sqlite3/storage.go | 1 + 7 files changed, 144 insertions(+), 3 deletions(-) create mode 100644 keyserver/storage/postgres/deltas/2022042612000000_xsigning_idx.go create mode 100644 keyserver/storage/sqlite3/deltas/2022042612000000_xsigning_idx.go diff --git a/keyserver/internal/cross_signing.go b/keyserver/internal/cross_signing.go index 2281f4bbf..08bbfedb8 100644 --- a/keyserver/internal/cross_signing.go +++ b/keyserver/internal/cross_signing.go @@ -362,6 +362,13 @@ func (a *KeyInternalAPI) processSelfSignatures( for targetKeyID, signature := range forTargetUserID { switch sig := signature.CrossSigningBody.(type) { case *gomatrixserverlib.CrossSigningKey: + for keyID := range sig.Keys { + split := strings.SplitN(string(keyID), ":", 2) + if len(split) > 1 && gomatrixserverlib.KeyID(split[1]) == targetKeyID { + targetKeyID = keyID // contains the ed25519: or other scheme + break + } + } for originUserID, forOriginUserID := range sig.Signatures { for originKeyID, originSig := range forOriginUserID { if err := a.DB.StoreCrossSigningSigsForTarget( diff --git a/keyserver/storage/postgres/cross_signing_sigs_table.go b/keyserver/storage/postgres/cross_signing_sigs_table.go index 40633c05c..b101e7ce5 100644 --- a/keyserver/storage/postgres/cross_signing_sigs_table.go +++ b/keyserver/storage/postgres/cross_signing_sigs_table.go @@ -33,8 +33,10 @@ CREATE TABLE IF NOT EXISTS keyserver_cross_signing_sigs ( target_user_id TEXT NOT NULL, target_key_id TEXT NOT NULL, signature TEXT NOT NULL, - PRIMARY KEY (origin_user_id, target_user_id, target_key_id) + PRIMARY KEY (origin_user_id, origin_key_id, target_user_id, target_key_id) ); + +CREATE INDEX IF NOT EXISTS keyserver_cross_signing_sigs_idx ON keyserver_cross_signing_sigs (origin_user_id, target_user_id, target_key_id); ` const selectCrossSigningSigsForTargetSQL = "" + @@ -44,7 +46,7 @@ const selectCrossSigningSigsForTargetSQL = "" + const upsertCrossSigningSigsForTargetSQL = "" + "INSERT INTO keyserver_cross_signing_sigs (origin_user_id, origin_key_id, target_user_id, target_key_id, signature)" + " VALUES($1, $2, $3, $4, $5)" + - " ON CONFLICT (origin_user_id, target_user_id, target_key_id) DO UPDATE SET (origin_key_id, signature) = ($2, $5)" + " ON CONFLICT (origin_user_id, origin_key_id, target_user_id, target_key_id) DO UPDATE SET signature = $5" const deleteCrossSigningSigsForTargetSQL = "" + "DELETE FROM keyserver_cross_signing_sigs WHERE target_user_id=$1 AND target_key_id=$2" diff --git a/keyserver/storage/postgres/deltas/2022042612000000_xsigning_idx.go b/keyserver/storage/postgres/deltas/2022042612000000_xsigning_idx.go new file mode 100644 index 000000000..12956e3b4 --- /dev/null +++ b/keyserver/storage/postgres/deltas/2022042612000000_xsigning_idx.go @@ -0,0 +1,52 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package deltas + +import ( + "database/sql" + "fmt" + + "github.com/matrix-org/dendrite/internal/sqlutil" +) + +func LoadFixCrossSigningSignatureIndexes(m *sqlutil.Migrations) { + m.AddMigration(UpFixCrossSigningSignatureIndexes, DownFixCrossSigningSignatureIndexes) +} + +func UpFixCrossSigningSignatureIndexes(tx *sql.Tx) error { + _, err := tx.Exec(` + ALTER TABLE keyserver_cross_signing_sigs DROP CONSTRAINT keyserver_cross_signing_sigs_pkey; + ALTER TABLE keyserver_cross_signing_sigs ADD PRIMARY KEY (origin_user_id, origin_key_id, target_user_id, target_key_id); + + CREATE INDEX IF NOT EXISTS keyserver_cross_signing_sigs_idx ON keyserver_cross_signing_sigs (origin_user_id, target_user_id, target_key_id); + `) + if err != nil { + return fmt.Errorf("failed to execute upgrade: %w", err) + } + return nil +} + +func DownFixCrossSigningSignatureIndexes(tx *sql.Tx) error { + _, err := tx.Exec(` + ALTER TABLE keyserver_cross_signing_sigs DROP CONSTRAINT keyserver_cross_signing_sigs_pkey; + ALTER TABLE keyserver_cross_signing_sigs ADD PRIMARY KEY (origin_user_id, target_user_id, target_key_id); + + DROP INDEX IF EXISTS keyserver_cross_signing_sigs_idx; + `) + if err != nil { + return fmt.Errorf("failed to execute downgrade: %w", err) + } + return nil +} diff --git a/keyserver/storage/postgres/storage.go b/keyserver/storage/postgres/storage.go index 136986885..d4c7e2cc7 100644 --- a/keyserver/storage/postgres/storage.go +++ b/keyserver/storage/postgres/storage.go @@ -54,6 +54,7 @@ func NewDatabase(dbProperties *config.DatabaseOptions) (*shared.Database, error) } m := sqlutil.NewMigrations() deltas.LoadRefactorKeyChanges(m) + deltas.LoadFixCrossSigningSignatureIndexes(m) if err = m.RunDeltas(db, dbProperties); err != nil { return nil, err } diff --git a/keyserver/storage/sqlite3/cross_signing_sigs_table.go b/keyserver/storage/sqlite3/cross_signing_sigs_table.go index 29ee889fd..36d562b8a 100644 --- a/keyserver/storage/sqlite3/cross_signing_sigs_table.go +++ b/keyserver/storage/sqlite3/cross_signing_sigs_table.go @@ -33,8 +33,10 @@ CREATE TABLE IF NOT EXISTS keyserver_cross_signing_sigs ( target_user_id TEXT NOT NULL, target_key_id TEXT NOT NULL, signature TEXT NOT NULL, - PRIMARY KEY (origin_user_id, target_user_id, target_key_id) + PRIMARY KEY (origin_user_id, origin_key_id, target_user_id, target_key_id) ); + +CREATE INDEX IF NOT EXISTS keyserver_cross_signing_sigs_idx ON keyserver_cross_signing_sigs (origin_user_id, target_user_id, target_key_id); ` const selectCrossSigningSigsForTargetSQL = "" + diff --git a/keyserver/storage/sqlite3/deltas/2022042612000000_xsigning_idx.go b/keyserver/storage/sqlite3/deltas/2022042612000000_xsigning_idx.go new file mode 100644 index 000000000..230e39fef --- /dev/null +++ b/keyserver/storage/sqlite3/deltas/2022042612000000_xsigning_idx.go @@ -0,0 +1,76 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package deltas + +import ( + "database/sql" + "fmt" + + "github.com/matrix-org/dendrite/internal/sqlutil" +) + +func LoadFixCrossSigningSignatureIndexes(m *sqlutil.Migrations) { + m.AddMigration(UpFixCrossSigningSignatureIndexes, DownFixCrossSigningSignatureIndexes) +} + +func UpFixCrossSigningSignatureIndexes(tx *sql.Tx) error { + _, err := tx.Exec(` + CREATE TABLE IF NOT EXISTS keyserver_cross_signing_sigs_tmp ( + origin_user_id TEXT NOT NULL, + origin_key_id TEXT NOT NULL, + target_user_id TEXT NOT NULL, + target_key_id TEXT NOT NULL, + signature TEXT NOT NULL, + PRIMARY KEY (origin_user_id, origin_key_id, target_user_id, target_key_id) + ); + + INSERT INTO keyserver_cross_signing_sigs_tmp (origin_user_id, origin_key_id, target_user_id, target_key_id, signature) + SELECT origin_user_id, origin_key_id, target_user_id, target_key_id, signature FROM keyserver_cross_signing_sigs; + + DROP TABLE keyserver_cross_signing_sigs; + ALTER TABLE keyserver_cross_signing_sigs_tmp RENAME TO keyserver_cross_signing_sigs; + + CREATE INDEX IF NOT EXISTS keyserver_cross_signing_sigs_idx ON keyserver_cross_signing_sigs (origin_user_id, target_user_id, target_key_id); + `) + if err != nil { + return fmt.Errorf("failed to execute upgrade: %w", err) + } + return nil +} + +func DownFixCrossSigningSignatureIndexes(tx *sql.Tx) error { + _, err := tx.Exec(` + CREATE TABLE IF NOT EXISTS keyserver_cross_signing_sigs_tmp ( + origin_user_id TEXT NOT NULL, + origin_key_id TEXT NOT NULL, + target_user_id TEXT NOT NULL, + target_key_id TEXT NOT NULL, + signature TEXT NOT NULL, + PRIMARY KEY (origin_user_id, target_user_id, target_key_id) + ); + + INSERT INTO keyserver_cross_signing_sigs_tmp (origin_user_id, origin_key_id, target_user_id, target_key_id, signature) + SELECT origin_user_id, origin_key_id, target_user_id, target_key_id, signature FROM keyserver_cross_signing_sigs; + + DROP TABLE keyserver_cross_signing_sigs; + ALTER TABLE keyserver_cross_signing_sigs_tmp RENAME TO keyserver_cross_signing_sigs; + + DELETE INDEX IF EXISTS keyserver_cross_signing_sigs_idx; + `) + if err != nil { + return fmt.Errorf("failed to execute downgrade: %w", err) + } + return nil +} diff --git a/keyserver/storage/sqlite3/storage.go b/keyserver/storage/sqlite3/storage.go index 0e0adceef..84d4cdf55 100644 --- a/keyserver/storage/sqlite3/storage.go +++ b/keyserver/storage/sqlite3/storage.go @@ -53,6 +53,7 @@ func NewDatabase(dbProperties *config.DatabaseOptions) (*shared.Database, error) m := sqlutil.NewMigrations() deltas.LoadRefactorKeyChanges(m) + deltas.LoadFixCrossSigningSignatureIndexes(m) if err = m.RunDeltas(db, dbProperties); err != nil { return nil, err } From 4c19f22725b8f534163ad37845650005b32172ad Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Tue, 26 Apr 2022 15:50:56 +0200 Subject: [PATCH 17/50] Fix account_data not correctly send in a complete sync (#2379) * Return the StreamPosition from the database and not the latest * Fix linter issue --- syncapi/storage/interface.go | 2 +- syncapi/storage/postgres/account_data_table.go | 18 +++++++++++------- syncapi/storage/shared/syncserver.go | 2 +- syncapi/storage/sqlite3/account_data_table.go | 18 +++++++++++------- syncapi/storage/tables/interface.go | 2 +- syncapi/streams/stream_accountdata.go | 4 ++-- 6 files changed, 27 insertions(+), 19 deletions(-) diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 0fea88da6..13065fa6b 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -81,7 +81,7 @@ type Database interface { // Returns a map following the format data[roomID] = []dataTypes // If no data is retrieved, returns an empty map // If there was an issue with the retrieval, returns an error - GetAccountDataInRange(ctx context.Context, userID string, r types.Range, accountDataFilterPart *gomatrixserverlib.EventFilter) (map[string][]string, error) + GetAccountDataInRange(ctx context.Context, userID string, r types.Range, accountDataFilterPart *gomatrixserverlib.EventFilter) (map[string][]string, types.StreamPosition, error) // UpsertAccountData keeps track of new or updated account data, by saving the type // of the new/updated data, and the user ID and room ID the data is related to (empty) // room ID means the data isn't specific to any room) diff --git a/syncapi/storage/postgres/account_data_table.go b/syncapi/storage/postgres/account_data_table.go index 25bdb1da3..22bb4d7fa 100644 --- a/syncapi/storage/postgres/account_data_table.go +++ b/syncapi/storage/postgres/account_data_table.go @@ -57,7 +57,7 @@ const insertAccountDataSQL = "" + " RETURNING id" const selectAccountDataInRangeSQL = "" + - "SELECT room_id, type FROM syncapi_account_data_type" + + "SELECT id, room_id, type FROM syncapi_account_data_type" + " WHERE user_id = $1 AND id > $2 AND id <= $3" + " AND ( $4::text[] IS NULL OR type LIKE ANY($4) )" + " AND ( $5::text[] IS NULL OR NOT(type LIKE ANY($5)) )" + @@ -103,7 +103,7 @@ func (s *accountDataStatements) SelectAccountDataInRange( userID string, r types.Range, accountDataEventFilter *gomatrixserverlib.EventFilter, -) (data map[string][]string, err error) { +) (data map[string][]string, pos types.StreamPosition, err error) { data = make(map[string][]string) rows, err := s.selectAccountDataInRangeStmt.QueryContext(ctx, userID, r.Low(), r.High(), @@ -116,11 +116,12 @@ func (s *accountDataStatements) SelectAccountDataInRange( } defer internal.CloseAndLogIfError(ctx, rows, "selectAccountDataInRange: rows.close() failed") - for rows.Next() { - var dataType string - var roomID string + var dataType string + var roomID string + var id types.StreamPosition - if err = rows.Scan(&roomID, &dataType); err != nil { + for rows.Next() { + if err = rows.Scan(&id, &roomID, &dataType); err != nil { return } @@ -129,8 +130,11 @@ func (s *accountDataStatements) SelectAccountDataInRange( } else { data[roomID] = []string{dataType} } + if id > pos { + pos = id + } } - return data, rows.Err() + return data, pos, rows.Err() } func (s *accountDataStatements) SelectMaxAccountDataID( diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index 3c431db48..69bceb624 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -265,7 +265,7 @@ func (d *Database) DeletePeeks( func (d *Database) GetAccountDataInRange( ctx context.Context, userID string, r types.Range, accountDataFilterPart *gomatrixserverlib.EventFilter, -) (map[string][]string, error) { +) (map[string][]string, types.StreamPosition, error) { return d.AccountData.SelectAccountDataInRange(ctx, userID, r, accountDataFilterPart) } diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go index 71a098177..e0d97ec32 100644 --- a/syncapi/storage/sqlite3/account_data_table.go +++ b/syncapi/storage/sqlite3/account_data_table.go @@ -43,7 +43,7 @@ const insertAccountDataSQL = "" + // further parameters are added by prepareWithFilters const selectAccountDataInRangeSQL = "" + - "SELECT room_id, type FROM syncapi_account_data_type" + + "SELECT id, room_id, type FROM syncapi_account_data_type" + " WHERE user_id = $1 AND id > $2 AND id <= $3" const selectMaxAccountDataIDSQL = "" + @@ -95,7 +95,7 @@ func (s *accountDataStatements) SelectAccountDataInRange( userID string, r types.Range, filter *gomatrixserverlib.EventFilter, -) (data map[string][]string, err error) { +) (data map[string][]string, pos types.StreamPosition, err error) { data = make(map[string][]string) stmt, params, err := prepareWithFilters( s.db, nil, selectAccountDataInRangeSQL, @@ -112,11 +112,12 @@ func (s *accountDataStatements) SelectAccountDataInRange( } defer internal.CloseAndLogIfError(ctx, rows, "selectAccountDataInRange: rows.close() failed") - for rows.Next() { - var dataType string - var roomID string + var dataType string + var roomID string + var id types.StreamPosition - if err = rows.Scan(&roomID, &dataType); err != nil { + for rows.Next() { + if err = rows.Scan(&id, &roomID, &dataType); err != nil { return } @@ -125,9 +126,12 @@ func (s *accountDataStatements) SelectAccountDataInRange( } else { data[roomID] = []string{dataType} } + if id > pos { + pos = id + } } - return data, nil + return data, pos, nil } func (s *accountDataStatements) SelectMaxAccountDataID( diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index ac713dd5c..32b1c34ef 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -27,7 +27,7 @@ import ( type AccountData interface { InsertAccountData(ctx context.Context, txn *sql.Tx, userID, roomID, dataType string) (pos types.StreamPosition, err error) // SelectAccountDataInRange returns a map of room ID to a list of `dataType`. - SelectAccountDataInRange(ctx context.Context, userID string, r types.Range, accountDataEventFilter *gomatrixserverlib.EventFilter) (data map[string][]string, err error) + SelectAccountDataInRange(ctx context.Context, userID string, r types.Range, accountDataEventFilter *gomatrixserverlib.EventFilter) (data map[string][]string, pos types.StreamPosition, err error) SelectMaxAccountDataID(ctx context.Context, txn *sql.Tx) (id int64, err error) } diff --git a/syncapi/streams/stream_accountdata.go b/syncapi/streams/stream_accountdata.go index 094c51485..99cd4a92a 100644 --- a/syncapi/streams/stream_accountdata.go +++ b/syncapi/streams/stream_accountdata.go @@ -43,7 +43,7 @@ func (p *AccountDataStreamProvider) IncrementalSync( To: to, } - dataTypes, err := p.DB.GetAccountDataInRange( + dataTypes, pos, err := p.DB.GetAccountDataInRange( ctx, req.Device.UserID, r, &req.Filter.AccountData, ) if err != nil { @@ -95,5 +95,5 @@ func (p *AccountDataStreamProvider) IncrementalSync( } } - return to + return pos } From 6892e0f0e02466be3cac6fc6f17267aeecb5961b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 26 Apr 2022 16:02:21 +0100 Subject: [PATCH 18/50] Start account data ID from `from` --- syncapi/storage/postgres/account_data_table.go | 2 +- syncapi/storage/sqlite3/account_data_table.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/syncapi/storage/postgres/account_data_table.go b/syncapi/storage/postgres/account_data_table.go index 22bb4d7fa..0a7146913 100644 --- a/syncapi/storage/postgres/account_data_table.go +++ b/syncapi/storage/postgres/account_data_table.go @@ -118,7 +118,7 @@ func (s *accountDataStatements) SelectAccountDataInRange( var dataType string var roomID string - var id types.StreamPosition + id := r.From for rows.Next() { if err = rows.Scan(&id, &roomID, &dataType); err != nil { diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go index e0d97ec32..d84159ac8 100644 --- a/syncapi/storage/sqlite3/account_data_table.go +++ b/syncapi/storage/sqlite3/account_data_table.go @@ -114,7 +114,7 @@ func (s *accountDataStatements) SelectAccountDataInRange( var dataType string var roomID string - var id types.StreamPosition + id := r.From for rows.Next() { if err = rows.Scan(&id, &roomID, &dataType); err != nil { From f6d07768a82cdea07c56cf4ae463449292fa9fe4 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 26 Apr 2022 16:07:13 +0100 Subject: [PATCH 19/50] Fix account data position --- syncapi/storage/postgres/account_data_table.go | 3 ++- syncapi/storage/sqlite3/account_data_table.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/syncapi/storage/postgres/account_data_table.go b/syncapi/storage/postgres/account_data_table.go index 0a7146913..ec1919fca 100644 --- a/syncapi/storage/postgres/account_data_table.go +++ b/syncapi/storage/postgres/account_data_table.go @@ -105,6 +105,7 @@ func (s *accountDataStatements) SelectAccountDataInRange( accountDataEventFilter *gomatrixserverlib.EventFilter, ) (data map[string][]string, pos types.StreamPosition, err error) { data = make(map[string][]string) + pos = r.Low() rows, err := s.selectAccountDataInRangeStmt.QueryContext(ctx, userID, r.Low(), r.High(), pq.StringArray(filterConvertTypeWildcardToSQL(accountDataEventFilter.Types)), @@ -118,7 +119,7 @@ func (s *accountDataStatements) SelectAccountDataInRange( var dataType string var roomID string - id := r.From + var id types.StreamPosition for rows.Next() { if err = rows.Scan(&id, &roomID, &dataType); err != nil { diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go index d84159ac8..2c7272ea8 100644 --- a/syncapi/storage/sqlite3/account_data_table.go +++ b/syncapi/storage/sqlite3/account_data_table.go @@ -96,6 +96,7 @@ func (s *accountDataStatements) SelectAccountDataInRange( r types.Range, filter *gomatrixserverlib.EventFilter, ) (data map[string][]string, pos types.StreamPosition, err error) { + pos = r.Low() data = make(map[string][]string) stmt, params, err := prepareWithFilters( s.db, nil, selectAccountDataInRangeSQL, @@ -114,7 +115,7 @@ func (s *accountDataStatements) SelectAccountDataInRange( var dataType string var roomID string - id := r.From + var id types.StreamPosition for rows.Next() { if err = rows.Scan(&id, &roomID, &dataType); err != nil { From b527e33c16d70ee6f94ac12c077b43283ff1fd86 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 26 Apr 2022 16:58:20 +0100 Subject: [PATCH 20/50] Send all account data on complete sync by default Squashed commit of the following: commit 0ec8de57261d573a5f88577aa9d7a1174d3999b9 Author: Neil Alexander Date: Tue Apr 26 16:56:30 2022 +0100 Select filter onto provided target filter commit da40b6fffbf5737864b223f49900048f557941f9 Author: Neil Alexander Date: Tue Apr 26 16:48:00 2022 +0100 Specify other field too commit ffc0b0801f63bb4d3061b6813e3ce5f3b4c8fbcb Author: Neil Alexander Date: Tue Apr 26 16:45:44 2022 +0100 Send as much account data as possible during complete sync --- syncapi/routing/filter.go | 4 ++-- syncapi/storage/interface.go | 6 +++--- syncapi/storage/postgres/filter_table.go | 13 ++++++------- syncapi/storage/shared/syncserver.go | 6 +++--- syncapi/storage/sqlite3/filter_table.go | 13 ++++++------- syncapi/storage/tables/interface.go | 2 +- syncapi/sync/request.go | 12 +++++++++--- 7 files changed, 30 insertions(+), 26 deletions(-) diff --git a/syncapi/routing/filter.go b/syncapi/routing/filter.go index baa4d841c..1a10bd649 100644 --- a/syncapi/routing/filter.go +++ b/syncapi/routing/filter.go @@ -44,8 +44,8 @@ func GetFilter( return jsonerror.InternalServerError() } - filter, err := syncDB.GetFilter(req.Context(), localpart, filterID) - if err != nil { + filter := gomatrixserverlib.DefaultFilter() + if err := syncDB.GetFilter(req.Context(), &filter, localpart, filterID); err != nil { //TODO better error handling. This error message is *probably* right, // but if there are obscure db errors, this will also be returned, // even though it is not correct. diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 13065fa6b..43aaa3588 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -125,10 +125,10 @@ type Database interface { // CleanSendToDeviceUpdates removes all send-to-device messages BEFORE the specified // from position, preventing the send-to-device table from growing indefinitely. CleanSendToDeviceUpdates(ctx context.Context, userID, deviceID string, before types.StreamPosition) (err error) - // GetFilter looks up the filter associated with a given local user and filter ID. - // Returns a filter structure. Otherwise returns an error if no such filter exists + // GetFilter looks up the filter associated with a given local user and filter ID + // and populates the target filter. Otherwise returns an error if no such filter exists // or if there was an error talking to the database. - GetFilter(ctx context.Context, localpart string, filterID string) (*gomatrixserverlib.Filter, error) + GetFilter(ctx context.Context, target *gomatrixserverlib.Filter, localpart string, filterID string) error // PutFilter puts the passed filter into the database. // Returns the filterID as a string. Otherwise returns an error if something // goes wrong. diff --git a/syncapi/storage/postgres/filter_table.go b/syncapi/storage/postgres/filter_table.go index dfd3d6963..c82ef092f 100644 --- a/syncapi/storage/postgres/filter_table.go +++ b/syncapi/storage/postgres/filter_table.go @@ -73,21 +73,20 @@ func NewPostgresFilterTable(db *sql.DB) (tables.Filter, error) { } func (s *filterStatements) SelectFilter( - ctx context.Context, localpart string, filterID string, -) (*gomatrixserverlib.Filter, error) { + ctx context.Context, target *gomatrixserverlib.Filter, localpart string, filterID string, +) error { // Retrieve filter from database (stored as canonical JSON) var filterData []byte err := s.selectFilterStmt.QueryRowContext(ctx, localpart, filterID).Scan(&filterData) if err != nil { - return nil, err + return err } // Unmarshal JSON into Filter struct - filter := gomatrixserverlib.DefaultFilter() - if err = json.Unmarshal(filterData, &filter); err != nil { - return nil, err + if err = json.Unmarshal(filterData, &target); err != nil { + return err } - return &filter, nil + return nil } func (s *filterStatements) InsertFilter( diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index 69bceb624..25aca50ae 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -513,9 +513,9 @@ func (d *Database) StreamToTopologicalPosition( } func (d *Database) GetFilter( - ctx context.Context, localpart string, filterID string, -) (*gomatrixserverlib.Filter, error) { - return d.Filter.SelectFilter(ctx, localpart, filterID) + ctx context.Context, target *gomatrixserverlib.Filter, localpart string, filterID string, +) error { + return d.Filter.SelectFilter(ctx, target, localpart, filterID) } func (d *Database) PutFilter( diff --git a/syncapi/storage/sqlite3/filter_table.go b/syncapi/storage/sqlite3/filter_table.go index 0cfebef2a..6081a48b1 100644 --- a/syncapi/storage/sqlite3/filter_table.go +++ b/syncapi/storage/sqlite3/filter_table.go @@ -77,21 +77,20 @@ func NewSqliteFilterTable(db *sql.DB) (tables.Filter, error) { } func (s *filterStatements) SelectFilter( - ctx context.Context, localpart string, filterID string, -) (*gomatrixserverlib.Filter, error) { + ctx context.Context, target *gomatrixserverlib.Filter, localpart string, filterID string, +) error { // Retrieve filter from database (stored as canonical JSON) var filterData []byte err := s.selectFilterStmt.QueryRowContext(ctx, localpart, filterID).Scan(&filterData) if err != nil { - return nil, err + return err } // Unmarshal JSON into Filter struct - filter := gomatrixserverlib.DefaultFilter() - if err = json.Unmarshal(filterData, &filter); err != nil { - return nil, err + if err = json.Unmarshal(filterData, &target); err != nil { + return err } - return &filter, nil + return nil } func (s *filterStatements) InsertFilter( diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index 32b1c34ef..4ff4689ed 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -157,7 +157,7 @@ type SendToDevice interface { } type Filter interface { - SelectFilter(ctx context.Context, localpart string, filterID string) (*gomatrixserverlib.Filter, error) + SelectFilter(ctx context.Context, target *gomatrixserverlib.Filter, localpart string, filterID string) error InsertFilter(ctx context.Context, filter *gomatrixserverlib.Filter, localpart string) (filterID string, err error) } diff --git a/syncapi/sync/request.go b/syncapi/sync/request.go index f04f172d3..c9ee8e4a8 100644 --- a/syncapi/sync/request.go +++ b/syncapi/sync/request.go @@ -18,6 +18,7 @@ import ( "database/sql" "encoding/json" "fmt" + "math" "net/http" "strconv" "time" @@ -47,6 +48,13 @@ func newSyncRequest(req *http.Request, device userapi.Device, syncDB storage.Dat } // TODO: read from stored filters too filter := gomatrixserverlib.DefaultFilter() + if since.IsEmpty() { + // Send as much account data down for complete syncs as possible + // by default, otherwise clients do weird things while waiting + // for the rest of the data to trickle down. + filter.AccountData.Limit = math.MaxInt + filter.Room.AccountData.Limit = math.MaxInt + } filterQuery := req.URL.Query().Get("filter") if filterQuery != "" { if filterQuery[0] == '{' { @@ -61,11 +69,9 @@ func newSyncRequest(req *http.Request, device userapi.Device, syncDB storage.Dat util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") return nil, fmt.Errorf("gomatrixserverlib.SplitID: %w", err) } - if f, err := syncDB.GetFilter(req.Context(), localpart, filterQuery); err != nil && err != sql.ErrNoRows { + if err := syncDB.GetFilter(req.Context(), &filter, localpart, filterQuery); err != nil && err != sql.ErrNoRows { util.GetLogger(req.Context()).WithError(err).Error("syncDB.GetFilter failed") return nil, fmt.Errorf("syncDB.GetFilter: %w", err) - } else if f != nil { - filter = *f } } } From 6c5c6d73d771e0ea5cc325e4251bcbfc48b7d55e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 26 Apr 2022 17:05:31 +0100 Subject: [PATCH 21/50] Use a value that is Go 1.16-friendly --- syncapi/sync/request.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/syncapi/sync/request.go b/syncapi/sync/request.go index c9ee8e4a8..9d4740e93 100644 --- a/syncapi/sync/request.go +++ b/syncapi/sync/request.go @@ -52,8 +52,8 @@ func newSyncRequest(req *http.Request, device userapi.Device, syncDB storage.Dat // Send as much account data down for complete syncs as possible // by default, otherwise clients do weird things while waiting // for the rest of the data to trickle down. - filter.AccountData.Limit = math.MaxInt - filter.Room.AccountData.Limit = math.MaxInt + filter.AccountData.Limit = math.MaxInt32 + filter.Room.AccountData.Limit = math.MaxInt32 } filterQuery := req.URL.Query().Get("filter") if filterQuery != "" { From 66b397b3c60c51bba36e4bce858733b2fda26f6a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 27 Apr 2022 11:25:07 +0100 Subject: [PATCH 22/50] Don't create fictitious presence entries (#2381) * Don't create fictitious presence entries for users that don't have any * Update whitelist, since that test probably shouldn't be passing * Fix panics --- syncapi/consumers/presence.go | 9 ++++++++- syncapi/storage/postgres/presence_table.go | 3 +++ syncapi/storage/sqlite3/presence_table.go | 3 +++ syncapi/streams/stream_presence.go | 12 +++++------ syncapi/sync/requestpool.go | 23 ++++++++++++---------- sytest-whitelist | 2 -- 6 files changed, 33 insertions(+), 19 deletions(-) diff --git a/syncapi/consumers/presence.go b/syncapi/consumers/presence.go index b198b2292..6bcca48f4 100644 --- a/syncapi/consumers/presence.go +++ b/syncapi/consumers/presence.go @@ -88,6 +88,11 @@ func (s *PresenceConsumer) Start() error { } return } + if presence == nil { + presence = &types.PresenceInternal{ + UserID: userID, + } + } deviceRes := api.QueryDevicesResponse{} if err = s.deviceAPI.QueryDevices(s.ctx, &api.QueryDevicesRequest{UserID: userID}, &deviceRes); err != nil { @@ -106,7 +111,9 @@ func (s *PresenceConsumer) Start() error { m.Header.Set(jetstream.UserID, presence.UserID) m.Header.Set("presence", presence.ClientFields.Presence) - m.Header.Set("status_msg", *presence.ClientFields.StatusMsg) + if presence.ClientFields.StatusMsg != nil { + m.Header.Set("status_msg", *presence.ClientFields.StatusMsg) + } m.Header.Set("last_active_ts", strconv.Itoa(int(presence.LastActiveTS))) if err = msg.RespondMsg(m); err != nil { diff --git a/syncapi/storage/postgres/presence_table.go b/syncapi/storage/postgres/presence_table.go index 49336c4eb..9f1e37f79 100644 --- a/syncapi/storage/postgres/presence_table.go +++ b/syncapi/storage/postgres/presence_table.go @@ -127,6 +127,9 @@ func (p *presenceStatements) GetPresenceForUser( } stmt := sqlutil.TxStmt(txn, p.selectPresenceForUsersStmt) err := stmt.QueryRowContext(ctx, userID).Scan(&result.Presence, &result.ClientFields.StatusMsg, &result.LastActiveTS) + if err == sql.ErrNoRows { + return nil, nil + } result.ClientFields.Presence = result.Presence.String() return result, err } diff --git a/syncapi/storage/sqlite3/presence_table.go b/syncapi/storage/sqlite3/presence_table.go index 00b16458d..177a01bf3 100644 --- a/syncapi/storage/sqlite3/presence_table.go +++ b/syncapi/storage/sqlite3/presence_table.go @@ -142,6 +142,9 @@ func (p *presenceStatements) GetPresenceForUser( } stmt := sqlutil.TxStmt(txn, p.selectPresenceForUsersStmt) err := stmt.QueryRowContext(ctx, userID).Scan(&result.Presence, &result.ClientFields.StatusMsg, &result.LastActiveTS) + if err == sql.ErrNoRows { + return nil, nil + } result.ClientFields.Presence = result.Presence.String() return result, err } diff --git a/syncapi/streams/stream_presence.go b/syncapi/streams/stream_presence.go index 9a6c5c130..614b88d48 100644 --- a/syncapi/streams/stream_presence.go +++ b/syncapi/streams/stream_presence.go @@ -16,7 +16,6 @@ package streams import ( "context" - "database/sql" "encoding/json" "sync" @@ -80,11 +79,10 @@ func (p *PresenceStreamProvider) IncrementalSync( if _, ok := presences[roomUsers[i]]; ok { continue } + // Bear in mind that this might return nil, but at least populating + // a nil means that there's a map entry so we won't repeat this call. presences[roomUsers[i]], err = p.DB.GetPresence(ctx, roomUsers[i]) if err != nil { - if err == sql.ErrNoRows { - continue - } req.Log.WithError(err).Error("unable to query presence for user") return from } @@ -93,8 +91,10 @@ func (p *PresenceStreamProvider) IncrementalSync( } lastPos := to - for i := range presences { - presence := presences[i] + for _, presence := range presences { + if presence == nil { + continue + } // Ignore users we don't share a room with if req.Device.UserID != presence.UserID && !p.notifier.IsSharedUser(req.Device.UserID, presence.UserID) { continue diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 703340997..76d550a65 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -127,14 +127,23 @@ func (rp *RequestPool) updatePresence(db storage.Presence, presence string, user if !ok { // this should almost never happen return } + newPresence := types.PresenceInternal{ - ClientFields: types.PresenceClientResponse{ - Presence: presenceID.String(), - }, Presence: presenceID, UserID: userID, LastActiveTS: gomatrixserverlib.AsTimestamp(time.Now()), } + + // ensure we also send the current status_msg to federated servers and not nil + dbPresence, err := db.GetPresence(context.Background(), userID) + if err != nil && err != sql.ErrNoRows { + return + } + if dbPresence != nil { + newPresence.ClientFields = dbPresence.ClientFields + } + newPresence.ClientFields.Presence = presenceID.String() + defer rp.presence.Store(userID, newPresence) // avoid spamming presence updates when syncing existingPresence, ok := rp.presence.LoadOrStore(userID, newPresence) @@ -145,13 +154,7 @@ func (rp *RequestPool) updatePresence(db storage.Presence, presence string, user } } - // ensure we also send the current status_msg to federated servers and not nil - dbPresence, err := db.GetPresence(context.Background(), userID) - if err != nil && err != sql.ErrNoRows { - return - } - - if err := rp.producer.SendPresence(userID, presenceID, dbPresence.ClientFields.StatusMsg); err != nil { + if err := rp.producer.SendPresence(userID, presenceID, newPresence.ClientFields.StatusMsg); err != nil { logrus.WithError(err).Error("Unable to publish presence message from sync") return } diff --git a/sytest-whitelist b/sytest-whitelist index c9829606f..6af8d89ff 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -681,8 +681,6 @@ GET /presence/:user_id/status fetches initial status PUT /presence/:user_id/status updates my presence Presence change reports an event to myself Existing members see new members' presence -#Existing members see new member's presence -Newly joined room includes presence in incremental sync Get presence for newly joined members in incremental sync User sees their own presence in a sync User sees updates to presence from other users in the incremental sync. From dca4afd2f0871ed53109121dff17048e69cc4935 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 27 Apr 2022 12:03:34 +0100 Subject: [PATCH 23/50] Don't send account data or receipts for left/forgotten rooms (#2382) * Only include account data and receipts for rooms in a complete sync that we care about * Fix global account data --- syncapi/streams/stream_accountdata.go | 6 ++++++ syncapi/streams/stream_receipt.go | 6 ++++++ syncapi/types/provider.go | 17 +++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/syncapi/streams/stream_accountdata.go b/syncapi/streams/stream_accountdata.go index 99cd4a92a..2cddbcf04 100644 --- a/syncapi/streams/stream_accountdata.go +++ b/syncapi/streams/stream_accountdata.go @@ -53,6 +53,12 @@ func (p *AccountDataStreamProvider) IncrementalSync( // Iterate over the rooms for roomID, dataTypes := range dataTypes { + // For a complete sync, make sure we're only including this room if + // that room was present in the joined rooms. + if from == 0 && roomID != "" && !req.IsRoomPresent(roomID) { + continue + } + // Request the missing data from the database for _, dataType := range dataTypes { dataReq := userapi.QueryAccountDataRequest{ diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go index 9d7d479a2..f4e84c7d0 100644 --- a/syncapi/streams/stream_receipt.go +++ b/syncapi/streams/stream_receipt.go @@ -62,6 +62,12 @@ func (p *ReceiptStreamProvider) IncrementalSync( } for roomID, receipts := range receiptsByRoom { + // For a complete sync, make sure we're only including this room if + // that room was present in the joined rooms. + if from == 0 && !req.IsRoomPresent(roomID) { + continue + } + jr := *types.NewJoinResponse() if existing, ok := req.Response.Rooms.Join[roomID]; ok { jr = existing diff --git a/syncapi/types/provider.go b/syncapi/types/provider.go index e6777f643..a9ea234d0 100644 --- a/syncapi/types/provider.go +++ b/syncapi/types/provider.go @@ -25,6 +25,23 @@ type SyncRequest struct { IgnoredUsers IgnoredUsers } +func (r *SyncRequest) IsRoomPresent(roomID string) bool { + membership, ok := r.Rooms[roomID] + if !ok { + return false + } + switch membership { + case gomatrixserverlib.Join: + return true + case gomatrixserverlib.Invite: + return true + case gomatrixserverlib.Peek: + return true + default: + return false + } +} + type StreamProvider interface { Setup() From 54ff4cf690918886c7e7a59a65ccff970c3aa1fc Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 27 Apr 2022 12:23:55 +0100 Subject: [PATCH 24/50] Don't try to federated-join via ourselves (#2383) --- federationapi/internal/perform.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go index 8cd944346..aac36cc76 100644 --- a/federationapi/internal/perform.go +++ b/federationapi/internal/perform.go @@ -75,7 +75,7 @@ func (r *FederationInternalAPI) PerformJoin( seenSet := make(map[gomatrixserverlib.ServerName]bool) var uniqueList []gomatrixserverlib.ServerName for _, srv := range request.ServerNames { - if seenSet[srv] { + if seenSet[srv] || srv == r.cfg.Matrix.ServerName { continue } seenSet[srv] = true From d7cc187ec00410b949ffae1625835f8ac9f36c29 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 27 Apr 2022 13:36:40 +0100 Subject: [PATCH 25/50] Prevent JetStream from handling OS signals, allow running as a Windows service (#2385) * Prevent JetStream from handling OS signals, allow running as a Windows service (fixes #2374) * Remove double import --- go.mod | 1 + go.sum | 1 + setup/base/base.go | 9 +++++++-- setup/jetstream/nats.go | 1 + 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index ba222ed8f..d51c3f75d 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/h2non/filetype v1.1.3 // indirect github.com/hashicorp/golang-lru v0.5.4 github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494 // indirect + github.com/kardianos/minwinsvc v1.0.0 // indirect github.com/lib/pq v1.10.5 github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 diff --git a/go.sum b/go.sum index 8bb306a82..f8daca79e 100644 --- a/go.sum +++ b/go.sum @@ -721,6 +721,7 @@ github.com/julienschmidt/httprouter v1.1.1-0.20151013225520-77a895ad01eb/go.mod github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/kardianos/minwinsvc v1.0.0 h1:+JfAi8IBJna0jY2dJGZqi7o15z13JelFIklJCAENALA= github.com/kardianos/minwinsvc v1.0.0/go.mod h1:Bgd0oc+D0Qo3bBytmNtyRKVlp85dAloLKhfxanPFFRc= github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8= github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE= diff --git a/setup/base/base.go b/setup/base/base.go index 43d613b0c..281153444 100644 --- a/setup/base/base.go +++ b/setup/base/base.go @@ -42,6 +42,7 @@ import ( userdb "github.com/matrix-org/dendrite/userapi/storage" "github.com/gorilla/mux" + "github.com/kardianos/minwinsvc" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" asinthttp "github.com/matrix-org/dendrite/appservice/inthttp" @@ -462,7 +463,8 @@ func (b *BaseDendrite) SetupAndServeHTTP( }() } - <-b.ProcessContext.WaitForShutdown() + minwinsvc.SetOnExit(b.ProcessContext.ShutdownDendrite) + b.WaitForShutdown() ctx, cancel := context.WithCancel(context.Background()) cancel() @@ -475,7 +477,10 @@ func (b *BaseDendrite) SetupAndServeHTTP( func (b *BaseDendrite) WaitForShutdown() { sigs := make(chan os.Signal, 1) signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) - <-sigs + select { + case <-sigs: + case <-b.ProcessContext.WaitForShutdown(): + } signal.Reset(syscall.SIGINT, syscall.SIGTERM) logrus.Warnf("Shutdown signal received") diff --git a/setup/jetstream/nats.go b/setup/jetstream/nats.go index 1c8a89e8d..8d5289697 100644 --- a/setup/jetstream/nats.go +++ b/setup/jetstream/nats.go @@ -44,6 +44,7 @@ func Prepare(process *process.ProcessContext, cfg *config.JetStream) (natsclient StoreDir: string(cfg.StoragePath), NoSystemAccount: true, MaxPayload: 16 * 1024 * 1024, + NoSigs: true, }) if err != nil { panic(err) From f023cdf8c42cc1a4bb850b478dbbf7d901b5e1bd Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 27 Apr 2022 15:05:49 +0200 Subject: [PATCH 26/50] Add UserAPI storage tests (#2384) * Add tests for parts of the userapi storage * Add tests for keybackup * Add LoginToken tests * Add OpenID tests * Add profile tests * Add pusher tests * Add ThreePID tests * Add notification tests * Add more device tests, fix numeric localpart query * Fix failing CI * Fix numeric local part query --- go.mod | 1 + setup/base/base.go | 5 +- userapi/storage/interface.go | 91 ++-- userapi/storage/postgres/accounts_table.go | 6 +- userapi/storage/postgres/devices_table.go | 22 +- userapi/storage/shared/storage.go | 15 - userapi/storage/sqlite3/accounts_table.go | 8 +- userapi/storage/sqlite3/devices_table.go | 22 +- userapi/storage/storage.go | 4 +- userapi/storage/storage_test.go | 539 +++++++++++++++++++++ userapi/storage/storage_wasm.go | 2 +- userapi/userapi_test.go | 2 +- 12 files changed, 640 insertions(+), 77 deletions(-) create mode 100644 userapi/storage/storage_test.go diff --git a/go.mod b/go.mod index d51c3f75d..a7caadfb5 100644 --- a/go.mod +++ b/go.mod @@ -47,6 +47,7 @@ require ( github.com/pressly/goose v2.7.0+incompatible github.com/prometheus/client_golang v1.12.1 github.com/sirupsen/logrus v1.8.1 + github.com/stretchr/testify v1.7.0 github.com/tidwall/gjson v1.14.0 github.com/tidwall/sjson v1.2.4 github.com/uber/jaeger-client-go v2.30.0+incompatible diff --git a/setup/base/base.go b/setup/base/base.go index 281153444..dbc5d2394 100644 --- a/setup/base/base.go +++ b/setup/base/base.go @@ -21,6 +21,7 @@ import ( "io" "net" "net/http" + _ "net/http/pprof" "os" "os/signal" "syscall" @@ -56,8 +57,6 @@ import ( userapi "github.com/matrix-org/dendrite/userapi/api" userapiinthttp "github.com/matrix-org/dendrite/userapi/inthttp" "github.com/sirupsen/logrus" - - _ "net/http/pprof" ) // BaseDendrite is a base for creating new instances of dendrite. It parses @@ -273,7 +272,7 @@ func (b *BaseDendrite) PushGatewayHTTPClient() pushgateway.Client { // CreateAccountsDB creates a new instance of the accounts database. Should only // be called once per component. func (b *BaseDendrite) CreateAccountsDB() userdb.Database { - db, err := userdb.NewDatabase( + db, err := userdb.NewUserAPIDatabase( &b.Cfg.UserAPI.AccountDatabase, b.Cfg.Global.ServerName, b.Cfg.UserAPI.BCryptCost, diff --git a/userapi/storage/interface.go b/userapi/storage/interface.go index b15470dd4..a4562cf19 100644 --- a/userapi/storage/interface.go +++ b/userapi/storage/interface.go @@ -27,18 +27,24 @@ import ( type Profile interface { GetProfileByLocalpart(ctx context.Context, localpart string) (*authtypes.Profile, error) SearchProfiles(ctx context.Context, searchString string, limit int) ([]authtypes.Profile, error) - SetPassword(ctx context.Context, localpart string, plaintextPassword string) error SetAvatarURL(ctx context.Context, localpart string, avatarURL string) error SetDisplayName(ctx context.Context, localpart string, displayName string) error } -type Database interface { - Profile - GetAccountByPassword(ctx context.Context, localpart, plaintextPassword string) (*api.Account, error) +type Account interface { // CreateAccount makes a new account with the given login name and password, and creates an empty profile // for this account. If no password is supplied, the account will be a passwordless account. If the // account already exists, it will return nil, ErrUserExists. CreateAccount(ctx context.Context, localpart string, plaintextPassword string, appserviceID string, accountType api.AccountType) (*api.Account, error) + GetAccountByPassword(ctx context.Context, localpart, plaintextPassword string) (*api.Account, error) + GetNewNumericLocalpart(ctx context.Context) (int64, error) + CheckAccountAvailability(ctx context.Context, localpart string) (bool, error) + GetAccountByLocalpart(ctx context.Context, localpart string) (*api.Account, error) + DeactivateAccount(ctx context.Context, localpart string) (err error) + SetPassword(ctx context.Context, localpart string, plaintextPassword string) error +} + +type AccountData interface { SaveAccountData(ctx context.Context, localpart, roomID, dataType string, content json.RawMessage) error GetAccountData(ctx context.Context, localpart string) (global map[string]json.RawMessage, rooms map[string]map[string]json.RawMessage, err error) // GetAccountDataByType returns account data matching a given @@ -46,26 +52,9 @@ type Database interface { // If no account data could be found, returns nil // Returns an error if there was an issue with the retrieval GetAccountDataByType(ctx context.Context, localpart, roomID, dataType string) (data json.RawMessage, err error) - GetNewNumericLocalpart(ctx context.Context) (int64, error) - SaveThreePIDAssociation(ctx context.Context, threepid, localpart, medium string) (err error) - RemoveThreePIDAssociation(ctx context.Context, threepid string, medium string) (err error) - GetLocalpartForThreePID(ctx context.Context, threepid string, medium string) (localpart string, err error) - GetThreePIDsForLocalpart(ctx context.Context, localpart string) (threepids []authtypes.ThreePID, err error) - CheckAccountAvailability(ctx context.Context, localpart string) (bool, error) - GetAccountByLocalpart(ctx context.Context, localpart string) (*api.Account, error) - DeactivateAccount(ctx context.Context, localpart string) (err error) - CreateOpenIDToken(ctx context.Context, token, localpart string) (exp int64, err error) - GetOpenIDTokenAttributes(ctx context.Context, token string) (*api.OpenIDTokenAttributes, error) - - // Key backups - CreateKeyBackup(ctx context.Context, userID, algorithm string, authData json.RawMessage) (version string, err error) - UpdateKeyBackupAuthData(ctx context.Context, userID, version string, authData json.RawMessage) (err error) - DeleteKeyBackup(ctx context.Context, userID, version string) (exists bool, err error) - GetKeyBackup(ctx context.Context, userID, version string) (versionResult, algorithm string, authData json.RawMessage, etag string, deleted bool, err error) - UpsertBackupKeys(ctx context.Context, version, userID string, uploads []api.InternalKeyBackupSession) (count int64, etag string, err error) - GetBackupKeys(ctx context.Context, version, userID, filterRoomID, filterSessionID string) (result map[string]map[string]api.KeyBackupSession, err error) - CountBackupKeys(ctx context.Context, version, userID string) (count int64, err error) +} +type Device interface { GetDeviceByAccessToken(ctx context.Context, token string) (*api.Device, error) GetDeviceByID(ctx context.Context, localpart, deviceID string) (*api.Device, error) GetDevicesByLocalpart(ctx context.Context, localpart string) ([]api.Device, error) @@ -79,11 +68,22 @@ type Database interface { CreateDevice(ctx context.Context, localpart string, deviceID *string, accessToken string, displayName *string, ipAddr, userAgent string) (dev *api.Device, returnErr error) UpdateDevice(ctx context.Context, localpart, deviceID string, displayName *string) error UpdateDeviceLastSeen(ctx context.Context, localpart, deviceID, ipAddr string) error - RemoveDevice(ctx context.Context, deviceID, localpart string) error RemoveDevices(ctx context.Context, localpart string, devices []string) error // RemoveAllDevices deleted all devices for this user. Returns the devices deleted. RemoveAllDevices(ctx context.Context, localpart, exceptDeviceID string) (devices []api.Device, err error) +} +type KeyBackup interface { + CreateKeyBackup(ctx context.Context, userID, algorithm string, authData json.RawMessage) (version string, err error) + UpdateKeyBackupAuthData(ctx context.Context, userID, version string, authData json.RawMessage) (err error) + DeleteKeyBackup(ctx context.Context, userID, version string) (exists bool, err error) + GetKeyBackup(ctx context.Context, userID, version string) (versionResult, algorithm string, authData json.RawMessage, etag string, deleted bool, err error) + UpsertBackupKeys(ctx context.Context, version, userID string, uploads []api.InternalKeyBackupSession) (count int64, etag string, err error) + GetBackupKeys(ctx context.Context, version, userID, filterRoomID, filterSessionID string) (result map[string]map[string]api.KeyBackupSession, err error) + CountBackupKeys(ctx context.Context, version, userID string) (count int64, err error) +} + +type LoginToken interface { // CreateLoginToken generates a token, stores and returns it. The lifetime is // determined by the loginTokenLifetime given to the Database constructor. CreateLoginToken(ctx context.Context, data *api.LoginTokenData) (*api.LoginTokenMetadata, error) @@ -94,21 +94,50 @@ type Database interface { // GetLoginTokenDataByToken returns the data associated with the given token. // May return sql.ErrNoRows. GetLoginTokenDataByToken(ctx context.Context, token string) (*api.LoginTokenData, error) +} - InsertNotification(ctx context.Context, localpart, eventID string, pos int64, tweaks map[string]interface{}, n *api.Notification) error - DeleteNotificationsUpTo(ctx context.Context, localpart, roomID string, pos int64) (affected bool, err error) - SetNotificationsRead(ctx context.Context, localpart, roomID string, pos int64, b bool) (affected bool, err error) - GetNotifications(ctx context.Context, localpart string, fromID int64, limit int, filter tables.NotificationFilter) ([]*api.Notification, int64, error) - GetNotificationCount(ctx context.Context, localpart string, filter tables.NotificationFilter) (int64, error) - GetRoomNotificationCounts(ctx context.Context, localpart, roomID string) (total int64, highlight int64, _ error) - DeleteOldNotifications(ctx context.Context) error +type OpenID interface { + CreateOpenIDToken(ctx context.Context, token, userID string) (exp int64, err error) + GetOpenIDTokenAttributes(ctx context.Context, token string) (*api.OpenIDTokenAttributes, error) +} +type Pusher interface { UpsertPusher(ctx context.Context, p api.Pusher, localpart string) error GetPushers(ctx context.Context, localpart string) ([]api.Pusher, error) RemovePusher(ctx context.Context, appid, pushkey, localpart string) error RemovePushers(ctx context.Context, appid, pushkey string) error } +type ThreePID interface { + SaveThreePIDAssociation(ctx context.Context, threepid, localpart, medium string) (err error) + RemoveThreePIDAssociation(ctx context.Context, threepid string, medium string) (err error) + GetLocalpartForThreePID(ctx context.Context, threepid string, medium string) (localpart string, err error) + GetThreePIDsForLocalpart(ctx context.Context, localpart string) (threepids []authtypes.ThreePID, err error) +} + +type Notification interface { + InsertNotification(ctx context.Context, localpart, eventID string, pos int64, tweaks map[string]interface{}, n *api.Notification) error + DeleteNotificationsUpTo(ctx context.Context, localpart, roomID string, pos int64) (affected bool, err error) + SetNotificationsRead(ctx context.Context, localpart, roomID string, pos int64, read bool) (affected bool, err error) + GetNotifications(ctx context.Context, localpart string, fromID int64, limit int, filter tables.NotificationFilter) ([]*api.Notification, int64, error) + GetNotificationCount(ctx context.Context, localpart string, filter tables.NotificationFilter) (int64, error) + GetRoomNotificationCounts(ctx context.Context, localpart, roomID string) (total int64, highlight int64, _ error) + DeleteOldNotifications(ctx context.Context) error +} + +type Database interface { + Account + AccountData + Device + KeyBackup + LoginToken + Notification + OpenID + Profile + Pusher + ThreePID +} + // Err3PIDInUse is the error returned when trying to save an association involving // a third-party identifier which is already associated to a local user. var Err3PIDInUse = errors.New("this third-party identifier is already in use") diff --git a/userapi/storage/postgres/accounts_table.go b/userapi/storage/postgres/accounts_table.go index 92311d56d..f86812f17 100644 --- a/userapi/storage/postgres/accounts_table.go +++ b/userapi/storage/postgres/accounts_table.go @@ -47,8 +47,6 @@ CREATE TABLE IF NOT EXISTS account_accounts ( -- TODO: -- upgraded_ts, devices, any email reset stuff? ); --- Create sequence for autogenerated numeric usernames -CREATE SEQUENCE IF NOT EXISTS numeric_username_seq START 1; ` const insertAccountSQL = "" + @@ -67,7 +65,7 @@ const selectPasswordHashSQL = "" + "SELECT password_hash FROM account_accounts WHERE localpart = $1 AND is_deactivated = FALSE" const selectNewNumericLocalpartSQL = "" + - "SELECT nextval('numeric_username_seq')" + "SELECT COALESCE(MAX(localpart::integer), 0) FROM account_accounts WHERE localpart ~ '^[0-9]*$'" type accountsStatements struct { insertAccountStmt *sql.Stmt @@ -178,5 +176,5 @@ func (s *accountsStatements) SelectNewNumericLocalpart( stmt = sqlutil.TxStmt(txn, stmt) } err = stmt.QueryRowContext(ctx).Scan(&id) - return + return id + 1, err } diff --git a/userapi/storage/postgres/devices_table.go b/userapi/storage/postgres/devices_table.go index 7bc5dc69b..fe8c54e04 100644 --- a/userapi/storage/postgres/devices_table.go +++ b/userapi/storage/postgres/devices_table.go @@ -78,7 +78,7 @@ const selectDeviceByIDSQL = "" + "SELECT display_name FROM device_devices WHERE localpart = $1 and device_id = $2" const selectDevicesByLocalpartSQL = "" + - "SELECT device_id, display_name, last_seen_ts, ip, user_agent FROM device_devices WHERE localpart = $1 AND device_id != $2" + "SELECT device_id, display_name, last_seen_ts, ip, user_agent FROM device_devices WHERE localpart = $1 AND device_id != $2 ORDER BY last_seen_ts DESC" const updateDeviceNameSQL = "" + "UPDATE device_devices SET display_name = $1 WHERE localpart = $2 AND device_id = $3" @@ -93,7 +93,7 @@ const deleteDevicesSQL = "" + "DELETE FROM device_devices WHERE localpart = $1 AND device_id = ANY($2)" const selectDevicesByIDSQL = "" + - "SELECT device_id, localpart, display_name FROM device_devices WHERE device_id = ANY($1)" + "SELECT device_id, localpart, display_name, last_seen_ts FROM device_devices WHERE device_id = ANY($1) ORDER BY last_seen_ts DESC" const updateDeviceLastSeen = "" + "UPDATE device_devices SET last_seen_ts = $1, ip = $2 WHERE localpart = $3 AND device_id = $4" @@ -235,16 +235,20 @@ func (s *devicesStatements) SelectDevicesByID(ctx context.Context, deviceIDs []s } defer internal.CloseAndLogIfError(ctx, rows, "selectDevicesByID: rows.close() failed") var devices []api.Device + var dev api.Device + var localpart string + var lastseents sql.NullInt64 + var displayName sql.NullString for rows.Next() { - var dev api.Device - var localpart string - var displayName sql.NullString - if err := rows.Scan(&dev.ID, &localpart, &displayName); err != nil { + if err := rows.Scan(&dev.ID, &localpart, &displayName, &lastseents); err != nil { return nil, err } if displayName.Valid { dev.DisplayName = displayName.String } + if lastseents.Valid { + dev.LastSeenTS = lastseents.Int64 + } dev.UserID = userutil.MakeUserID(localpart, s.serverName) devices = append(devices, dev) } @@ -262,10 +266,10 @@ func (s *devicesStatements) SelectDevicesByLocalpart( } defer internal.CloseAndLogIfError(ctx, rows, "selectDevicesByLocalpart: rows.close() failed") + var dev api.Device + var lastseents sql.NullInt64 + var id, displayname, ip, useragent sql.NullString for rows.Next() { - var dev api.Device - var lastseents sql.NullInt64 - var id, displayname, ip, useragent sql.NullString err = rows.Scan(&id, &displayname, &lastseents, &ip, &useragent) if err != nil { return devices, err diff --git a/userapi/storage/shared/storage.go b/userapi/storage/shared/storage.go index 72ae96ecc..f7212e030 100644 --- a/userapi/storage/shared/storage.go +++ b/userapi/storage/shared/storage.go @@ -577,21 +577,6 @@ func (d *Database) UpdateDevice( }) } -// RemoveDevice revokes a device by deleting the entry in the database -// matching with the given device ID and user ID localpart. -// If the device doesn't exist, it will not return an error -// If something went wrong during the deletion, it will return the SQL error. -func (d *Database) RemoveDevice( - ctx context.Context, deviceID, localpart string, -) error { - return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - if err := d.Devices.DeleteDevice(ctx, txn, deviceID, localpart); err != sql.ErrNoRows { - return err - } - return nil - }) -} - // RemoveDevices revokes one or more devices by deleting the entry in the database // matching with the given device IDs and user ID localpart. // If the devices don't exist, it will not return an error diff --git a/userapi/storage/sqlite3/accounts_table.go b/userapi/storage/sqlite3/accounts_table.go index e6c37e58e..6c5fe3071 100644 --- a/userapi/storage/sqlite3/accounts_table.go +++ b/userapi/storage/sqlite3/accounts_table.go @@ -65,7 +65,7 @@ const selectPasswordHashSQL = "" + "SELECT password_hash FROM account_accounts WHERE localpart = $1 AND is_deactivated = 0" const selectNewNumericLocalpartSQL = "" + - "SELECT COUNT(localpart) FROM account_accounts" + "SELECT COALESCE(MAX(CAST(localpart AS INT)), 0) FROM account_accounts WHERE CAST(localpart AS INT) <> 0" type accountsStatements struct { db *sql.DB @@ -121,6 +121,7 @@ func (s *accountsStatements) InsertAccount( UserID: userutil.MakeUserID(localpart, s.serverName), ServerName: s.serverName, AppServiceID: appserviceID, + AccountType: accountType, }, nil } @@ -177,5 +178,8 @@ func (s *accountsStatements) SelectNewNumericLocalpart( stmt = sqlutil.TxStmt(txn, stmt) } err = stmt.QueryRowContext(ctx).Scan(&id) - return + if err == sql.ErrNoRows { + return 1, nil + } + return id + 1, err } diff --git a/userapi/storage/sqlite3/devices_table.go b/userapi/storage/sqlite3/devices_table.go index 423640e90..7860bd6a2 100644 --- a/userapi/storage/sqlite3/devices_table.go +++ b/userapi/storage/sqlite3/devices_table.go @@ -63,7 +63,7 @@ const selectDeviceByIDSQL = "" + "SELECT display_name FROM device_devices WHERE localpart = $1 and device_id = $2" const selectDevicesByLocalpartSQL = "" + - "SELECT device_id, display_name, last_seen_ts, ip, user_agent FROM device_devices WHERE localpart = $1 AND device_id != $2" + "SELECT device_id, display_name, last_seen_ts, ip, user_agent FROM device_devices WHERE localpart = $1 AND device_id != $2 ORDER BY last_seen_ts DESC" const updateDeviceNameSQL = "" + "UPDATE device_devices SET display_name = $1 WHERE localpart = $2 AND device_id = $3" @@ -78,7 +78,7 @@ const deleteDevicesSQL = "" + "DELETE FROM device_devices WHERE localpart = $1 AND device_id IN ($2)" const selectDevicesByIDSQL = "" + - "SELECT device_id, localpart, display_name FROM device_devices WHERE device_id IN ($1)" + "SELECT device_id, localpart, display_name, last_seen_ts FROM device_devices WHERE device_id IN ($1) ORDER BY last_seen_ts DESC" const updateDeviceLastSeen = "" + "UPDATE device_devices SET last_seen_ts = $1, ip = $2 WHERE localpart = $3 AND device_id = $4" @@ -235,10 +235,10 @@ func (s *devicesStatements) SelectDevicesByLocalpart( return devices, err } + var dev api.Device + var lastseents sql.NullInt64 + var id, displayname, ip, useragent sql.NullString for rows.Next() { - var dev api.Device - var lastseents sql.NullInt64 - var id, displayname, ip, useragent sql.NullString err = rows.Scan(&id, &displayname, &lastseents, &ip, &useragent) if err != nil { return devices, err @@ -279,16 +279,20 @@ func (s *devicesStatements) SelectDevicesByID(ctx context.Context, deviceIDs []s } defer internal.CloseAndLogIfError(ctx, rows, "selectDevicesByID: rows.close() failed") var devices []api.Device + var dev api.Device + var localpart string + var displayName sql.NullString + var lastseents sql.NullInt64 for rows.Next() { - var dev api.Device - var localpart string - var displayName sql.NullString - if err := rows.Scan(&dev.ID, &localpart, &displayName); err != nil { + if err := rows.Scan(&dev.ID, &localpart, &displayName, &lastseents); err != nil { return nil, err } if displayName.Valid { dev.DisplayName = displayName.String } + if lastseents.Valid { + dev.LastSeenTS = lastseents.Int64 + } dev.UserID = userutil.MakeUserID(localpart, s.serverName) devices = append(devices, dev) } diff --git a/userapi/storage/storage.go b/userapi/storage/storage.go index f372fe7dc..faf1ce75c 100644 --- a/userapi/storage/storage.go +++ b/userapi/storage/storage.go @@ -28,9 +28,9 @@ import ( "github.com/matrix-org/dendrite/userapi/storage/sqlite3" ) -// NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) +// NewUserAPIDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) // and sets postgres connection parameters -func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (Database, error) { +func NewUserAPIDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (Database, error) { switch { case dbProperties.ConnectionString.IsSQLite(): return sqlite3.NewDatabase(dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart) diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go new file mode 100644 index 000000000..e6c7d35fc --- /dev/null +++ b/userapi/storage/storage_test.go @@ -0,0 +1,539 @@ +package storage_test + +import ( + "context" + "encoding/json" + "fmt" + "testing" + "time" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/internal/pushrules" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage" + "github.com/matrix-org/dendrite/userapi/storage/tables" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/stretchr/testify/assert" + "golang.org/x/crypto/bcrypt" +) + +const loginTokenLifetime = time.Minute + +var ( + openIDLifetimeMS = time.Minute.Milliseconds() + ctx = context.Background() +) + +func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) { + connStr, close := test.PrepareDBConnectionString(t, dbType) + db, err := storage.NewUserAPIDatabase(&config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + }, "localhost", bcrypt.MinCost, openIDLifetimeMS, loginTokenLifetime, "_server") + if err != nil { + t.Fatalf("NewUserAPIDatabase returned %s", err) + } + return db, close +} + +// Tests storing and getting account data +func Test_AccountData(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, close := mustCreateDatabase(t, dbType) + defer close() + alice := test.NewUser() + localpart, _, err := gomatrixserverlib.SplitID('@', alice.ID) + assert.NoError(t, err) + + room := test.NewRoom(t, alice) + events := room.Events() + + contentRoom := json.RawMessage(fmt.Sprintf(`{"event_id":"%s"}`, events[len(events)-1].EventID())) + err = db.SaveAccountData(ctx, localpart, room.ID, "m.fully_read", contentRoom) + assert.NoError(t, err, "unable to save account data") + + contentGlobal := json.RawMessage(fmt.Sprintf(`{"recent_rooms":["%s"]}`, room.ID)) + err = db.SaveAccountData(ctx, localpart, "", "im.vector.setting.breadcrumbs", contentGlobal) + assert.NoError(t, err, "unable to save account data") + + accountData, err := db.GetAccountDataByType(ctx, localpart, room.ID, "m.fully_read") + assert.NoError(t, err, "unable to get account data by type") + assert.Equal(t, contentRoom, accountData) + + globalData, roomData, err := db.GetAccountData(ctx, localpart) + assert.NoError(t, err) + assert.Equal(t, contentRoom, roomData[room.ID]["m.fully_read"]) + assert.Equal(t, contentGlobal, globalData["im.vector.setting.breadcrumbs"]) + }) +} + +// Tests the creation of accounts +func Test_Accounts(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, close := mustCreateDatabase(t, dbType) + defer close() + alice := test.NewUser() + aliceLocalpart, _, err := gomatrixserverlib.SplitID('@', alice.ID) + assert.NoError(t, err) + + accAlice, err := db.CreateAccount(ctx, aliceLocalpart, "testing", "", api.AccountTypeAdmin) + assert.NoError(t, err, "failed to create account") + // verify the newly create account is the same as returned by CreateAccount + var accGet *api.Account + accGet, err = db.GetAccountByPassword(ctx, aliceLocalpart, "testing") + assert.NoError(t, err, "failed to get account by password") + assert.Equal(t, accAlice, accGet) + accGet, err = db.GetAccountByLocalpart(ctx, aliceLocalpart) + assert.NoError(t, err, "failed to get account by localpart") + assert.Equal(t, accAlice, accGet) + + // check account availability + available, err := db.CheckAccountAvailability(ctx, aliceLocalpart) + assert.NoError(t, err, "failed to checkout account availability") + assert.Equal(t, false, available) + + available, err = db.CheckAccountAvailability(ctx, "unusedname") + assert.NoError(t, err, "failed to checkout account availability") + assert.Equal(t, true, available) + + // get guest account numeric aliceLocalpart + first, err := db.GetNewNumericLocalpart(ctx) + assert.NoError(t, err, "failed to get new numeric localpart") + // Create a new account to verify the numeric localpart is updated + _, err = db.CreateAccount(ctx, "", "testing", "", api.AccountTypeGuest) + assert.NoError(t, err, "failed to create account") + second, err := db.GetNewNumericLocalpart(ctx) + assert.NoError(t, err) + assert.Greater(t, second, first) + + // update password for alice + err = db.SetPassword(ctx, aliceLocalpart, "newPassword") + assert.NoError(t, err, "failed to update password") + accGet, err = db.GetAccountByPassword(ctx, aliceLocalpart, "newPassword") + assert.NoError(t, err, "failed to get account by new password") + assert.Equal(t, accAlice, accGet) + + // deactivate account + err = db.DeactivateAccount(ctx, aliceLocalpart) + assert.NoError(t, err, "failed to deactivate account") + // This should fail now, as the account is deactivated + _, err = db.GetAccountByPassword(ctx, aliceLocalpart, "newPassword") + assert.Error(t, err, "expected an error, got none") + + _, err = db.GetAccountByLocalpart(ctx, "unusename") + assert.Error(t, err, "expected an error for non existent localpart") + }) +} + +func Test_Devices(t *testing.T) { + alice := test.NewUser() + localpart, _, err := gomatrixserverlib.SplitID('@', alice.ID) + assert.NoError(t, err) + deviceID := util.RandomString(8) + accessToken := util.RandomString(16) + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, close := mustCreateDatabase(t, dbType) + defer close() + + deviceWithID, err := db.CreateDevice(ctx, localpart, &deviceID, accessToken, nil, "", "") + assert.NoError(t, err, "unable to create deviceWithoutID") + + gotDevice, err := db.GetDeviceByID(ctx, localpart, deviceID) + assert.NoError(t, err, "unable to get device by id") + assert.Equal(t, deviceWithID.ID, gotDevice.ID) // GetDeviceByID doesn't populate all fields + + gotDeviceAccessToken, err := db.GetDeviceByAccessToken(ctx, accessToken) + assert.NoError(t, err, "unable to get device by access token") + assert.Equal(t, deviceWithID.ID, gotDeviceAccessToken.ID) // GetDeviceByAccessToken doesn't populate all fields + + // create a device without existing device ID + accessToken = util.RandomString(16) + deviceWithoutID, err := db.CreateDevice(ctx, localpart, nil, accessToken, nil, "", "") + assert.NoError(t, err, "unable to create deviceWithoutID") + gotDeviceWithoutID, err := db.GetDeviceByID(ctx, localpart, deviceWithoutID.ID) + assert.NoError(t, err, "unable to get device by id") + assert.Equal(t, deviceWithoutID.ID, gotDeviceWithoutID.ID) // GetDeviceByID doesn't populate all fields + + // Get devices + devices, err := db.GetDevicesByLocalpart(ctx, localpart) + assert.NoError(t, err, "unable to get devices by localpart") + assert.Equal(t, 2, len(devices)) + deviceIDs := make([]string, 0, len(devices)) + for _, dev := range devices { + deviceIDs = append(deviceIDs, dev.ID) + } + + devices2, err := db.GetDevicesByID(ctx, deviceIDs) + assert.NoError(t, err, "unable to get devices by id") + assert.Equal(t, devices, devices2) + + // Update device + newName := "new display name" + err = db.UpdateDevice(ctx, localpart, deviceWithID.ID, &newName) + assert.NoError(t, err, "unable to update device displayname") + err = db.UpdateDeviceLastSeen(ctx, localpart, deviceWithID.ID, "127.0.0.1") + assert.NoError(t, err, "unable to update device last seen") + + deviceWithID.DisplayName = newName + deviceWithID.LastSeenIP = "127.0.0.1" + deviceWithID.LastSeenTS = int64(gomatrixserverlib.AsTimestamp(time.Now().Truncate(time.Second))) + devices, err = db.GetDevicesByLocalpart(ctx, localpart) + assert.NoError(t, err, "unable to get device by id") + assert.Equal(t, 2, len(devices)) + assert.Equal(t, deviceWithID.DisplayName, devices[0].DisplayName) + assert.Equal(t, deviceWithID.LastSeenIP, devices[0].LastSeenIP) + truncatedTime := gomatrixserverlib.Timestamp(devices[0].LastSeenTS).Time().Truncate(time.Second) + assert.Equal(t, gomatrixserverlib.Timestamp(deviceWithID.LastSeenTS), gomatrixserverlib.AsTimestamp(truncatedTime)) + + // create one more device and remove the devices step by step + newDeviceID := util.RandomString(16) + accessToken = util.RandomString(16) + _, err = db.CreateDevice(ctx, localpart, &newDeviceID, accessToken, nil, "", "") + assert.NoError(t, err, "unable to create new device") + + devices, err = db.GetDevicesByLocalpart(ctx, localpart) + assert.NoError(t, err, "unable to get device by id") + assert.Equal(t, 3, len(devices)) + + err = db.RemoveDevices(ctx, localpart, deviceIDs) + assert.NoError(t, err, "unable to remove devices") + devices, err = db.GetDevicesByLocalpart(ctx, localpart) + assert.NoError(t, err, "unable to get device by id") + assert.Equal(t, 1, len(devices)) + + deleted, err := db.RemoveAllDevices(ctx, localpart, "") + assert.NoError(t, err, "unable to remove all devices") + assert.Equal(t, 1, len(deleted)) + assert.Equal(t, newDeviceID, deleted[0].ID) + }) +} + +func Test_KeyBackup(t *testing.T) { + alice := test.NewUser() + room := test.NewRoom(t, alice) + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, close := mustCreateDatabase(t, dbType) + defer close() + + wantAuthData := json.RawMessage("my auth data") + wantVersion, err := db.CreateKeyBackup(ctx, alice.ID, "dummyAlgo", wantAuthData) + assert.NoError(t, err, "unable to create key backup") + // get key backup by version + gotVersion, gotAlgo, gotAuthData, _, _, err := db.GetKeyBackup(ctx, alice.ID, wantVersion) + assert.NoError(t, err, "unable to get key backup") + assert.Equal(t, wantVersion, gotVersion, "backup version mismatch") + assert.Equal(t, "dummyAlgo", gotAlgo, "backup algorithm mismatch") + assert.Equal(t, wantAuthData, gotAuthData, "backup auth data mismatch") + + // get any key backup + gotVersion, gotAlgo, gotAuthData, _, _, err = db.GetKeyBackup(ctx, alice.ID, "") + assert.NoError(t, err, "unable to get key backup") + assert.Equal(t, wantVersion, gotVersion, "backup version mismatch") + assert.Equal(t, "dummyAlgo", gotAlgo, "backup algorithm mismatch") + assert.Equal(t, wantAuthData, gotAuthData, "backup auth data mismatch") + + err = db.UpdateKeyBackupAuthData(ctx, alice.ID, wantVersion, json.RawMessage("my updated auth data")) + assert.NoError(t, err, "unable to update key backup auth data") + + uploads := []api.InternalKeyBackupSession{ + { + KeyBackupSession: api.KeyBackupSession{ + IsVerified: true, + SessionData: wantAuthData, + }, + RoomID: room.ID, + SessionID: "1", + }, + { + KeyBackupSession: api.KeyBackupSession{}, + RoomID: room.ID, + SessionID: "2", + }, + } + count, _, err := db.UpsertBackupKeys(ctx, wantVersion, alice.ID, uploads) + assert.NoError(t, err, "unable to upsert backup keys") + assert.Equal(t, int64(len(uploads)), count, "unexpected backup count") + + // do it again to update a key + uploads[1].IsVerified = true + count, _, err = db.UpsertBackupKeys(ctx, wantVersion, alice.ID, uploads[1:]) + assert.NoError(t, err, "unable to upsert backup keys") + assert.Equal(t, int64(len(uploads)), count, "unexpected backup count") + + // get backup keys by session id + gotBackupKeys, err := db.GetBackupKeys(ctx, wantVersion, alice.ID, room.ID, "1") + assert.NoError(t, err, "unable to get backup keys") + assert.Equal(t, uploads[0].KeyBackupSession, gotBackupKeys[room.ID]["1"]) + + // get backup keys by room id + gotBackupKeys, err = db.GetBackupKeys(ctx, wantVersion, alice.ID, room.ID, "") + assert.NoError(t, err, "unable to get backup keys") + assert.Equal(t, uploads[0].KeyBackupSession, gotBackupKeys[room.ID]["1"]) + + gotCount, err := db.CountBackupKeys(ctx, wantVersion, alice.ID) + assert.NoError(t, err, "unable to get backup keys count") + assert.Equal(t, count, gotCount, "unexpected backup count") + + // finally delete a key + exists, err := db.DeleteKeyBackup(ctx, alice.ID, wantVersion) + assert.NoError(t, err, "unable to delete key backup") + assert.True(t, exists) + + // this key should not exist + exists, err = db.DeleteKeyBackup(ctx, alice.ID, "3") + assert.NoError(t, err, "unable to delete key backup") + assert.False(t, exists) + }) +} + +func Test_LoginToken(t *testing.T) { + alice := test.NewUser() + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, close := mustCreateDatabase(t, dbType) + defer close() + + // create a new token + wantLoginToken := &api.LoginTokenData{UserID: alice.ID} + + gotMetadata, err := db.CreateLoginToken(ctx, wantLoginToken) + assert.NoError(t, err, "unable to create login token") + assert.NotNil(t, gotMetadata) + assert.Equal(t, time.Now().Add(loginTokenLifetime).Truncate(loginTokenLifetime), gotMetadata.Expiration.Truncate(loginTokenLifetime)) + + // get the new token + gotLoginToken, err := db.GetLoginTokenDataByToken(ctx, gotMetadata.Token) + assert.NoError(t, err, "unable to get login token") + assert.NotNil(t, gotLoginToken) + assert.Equal(t, wantLoginToken, gotLoginToken, "unexpected login token") + + // remove the login token again + err = db.RemoveLoginToken(ctx, gotMetadata.Token) + assert.NoError(t, err, "unable to remove login token") + + // check if the token was actually deleted + _, err = db.GetLoginTokenDataByToken(ctx, gotMetadata.Token) + assert.Error(t, err, "expected an error, but got none") + }) +} + +func Test_OpenID(t *testing.T) { + alice := test.NewUser() + token := util.RandomString(24) + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, close := mustCreateDatabase(t, dbType) + defer close() + + expiresAtMS := time.Now().UnixNano()/int64(time.Millisecond) + openIDLifetimeMS + expires, err := db.CreateOpenIDToken(ctx, token, alice.ID) + assert.NoError(t, err, "unable to create OpenID token") + assert.Equal(t, expiresAtMS, expires) + + attributes, err := db.GetOpenIDTokenAttributes(ctx, token) + assert.NoError(t, err, "unable to get OpenID token attributes") + assert.Equal(t, alice.ID, attributes.UserID) + assert.Equal(t, expiresAtMS, attributes.ExpiresAtMS) + }) +} + +func Test_Profile(t *testing.T) { + alice := test.NewUser() + aliceLocalpart, _, err := gomatrixserverlib.SplitID('@', alice.ID) + assert.NoError(t, err) + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, close := mustCreateDatabase(t, dbType) + defer close() + + // create account, which also creates a profile + _, err = db.CreateAccount(ctx, aliceLocalpart, "testing", "", api.AccountTypeAdmin) + assert.NoError(t, err, "failed to create account") + + gotProfile, err := db.GetProfileByLocalpart(ctx, aliceLocalpart) + assert.NoError(t, err, "unable to get profile by localpart") + wantProfile := &authtypes.Profile{Localpart: aliceLocalpart} + assert.Equal(t, wantProfile, gotProfile) + + // set avatar & displayname + wantProfile.DisplayName = "Alice" + wantProfile.AvatarURL = "mxc://aliceAvatar" + err = db.SetDisplayName(ctx, aliceLocalpart, "Alice") + assert.NoError(t, err, "unable to set displayname") + err = db.SetAvatarURL(ctx, aliceLocalpart, "mxc://aliceAvatar") + assert.NoError(t, err, "unable to set avatar url") + // verify profile + gotProfile, err = db.GetProfileByLocalpart(ctx, aliceLocalpart) + assert.NoError(t, err, "unable to get profile by localpart") + assert.Equal(t, wantProfile, gotProfile) + + // search profiles + searchRes, err := db.SearchProfiles(ctx, "Alice", 2) + assert.NoError(t, err, "unable to search profiles") + assert.Equal(t, 1, len(searchRes)) + assert.Equal(t, *wantProfile, searchRes[0]) + }) +} + +func Test_Pusher(t *testing.T) { + alice := test.NewUser() + aliceLocalpart, _, err := gomatrixserverlib.SplitID('@', alice.ID) + assert.NoError(t, err) + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, close := mustCreateDatabase(t, dbType) + defer close() + + appID := util.RandomString(8) + var pushKeys []string + var gotPushers []api.Pusher + for i := 0; i < 2; i++ { + pushKey := util.RandomString(8) + + wantPusher := api.Pusher{ + PushKey: pushKey, + Kind: api.HTTPKind, + AppID: appID, + AppDisplayName: util.RandomString(8), + DeviceDisplayName: util.RandomString(8), + ProfileTag: util.RandomString(8), + Language: util.RandomString(2), + } + err = db.UpsertPusher(ctx, wantPusher, aliceLocalpart) + assert.NoError(t, err, "unable to upsert pusher") + + // check it was actually persisted + gotPushers, err = db.GetPushers(ctx, aliceLocalpart) + assert.NoError(t, err, "unable to get pushers") + assert.Equal(t, i+1, len(gotPushers)) + assert.Equal(t, wantPusher, gotPushers[i]) + pushKeys = append(pushKeys, pushKey) + } + + // remove single pusher + err = db.RemovePusher(ctx, appID, pushKeys[0], aliceLocalpart) + assert.NoError(t, err, "unable to remove pusher") + gotPushers, err := db.GetPushers(ctx, aliceLocalpart) + assert.NoError(t, err, "unable to get pushers") + assert.Equal(t, 1, len(gotPushers)) + + // remove last pusher + err = db.RemovePushers(ctx, appID, pushKeys[1]) + assert.NoError(t, err, "unable to remove pusher") + gotPushers, err = db.GetPushers(ctx, aliceLocalpart) + assert.NoError(t, err, "unable to get pushers") + assert.Equal(t, 0, len(gotPushers)) + }) +} + +func Test_ThreePID(t *testing.T) { + alice := test.NewUser() + aliceLocalpart, _, err := gomatrixserverlib.SplitID('@', alice.ID) + assert.NoError(t, err) + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, close := mustCreateDatabase(t, dbType) + defer close() + threePID := util.RandomString(8) + medium := util.RandomString(8) + err = db.SaveThreePIDAssociation(ctx, threePID, aliceLocalpart, medium) + assert.NoError(t, err, "unable to save threepid association") + + // get the stored threepid + gotLocalpart, err := db.GetLocalpartForThreePID(ctx, threePID, medium) + assert.NoError(t, err, "unable to get localpart for threepid") + assert.Equal(t, aliceLocalpart, gotLocalpart) + + threepids, err := db.GetThreePIDsForLocalpart(ctx, aliceLocalpart) + assert.NoError(t, err, "unable to get threepids for localpart") + assert.Equal(t, 1, len(threepids)) + assert.Equal(t, authtypes.ThreePID{ + Address: threePID, + Medium: medium, + }, threepids[0]) + + // remove threepid association + err = db.RemoveThreePIDAssociation(ctx, threePID, medium) + assert.NoError(t, err, "unexpected error") + + // verify it was deleted + threepids, err = db.GetThreePIDsForLocalpart(ctx, aliceLocalpart) + assert.NoError(t, err, "unable to get threepids for localpart") + assert.Equal(t, 0, len(threepids)) + }) +} + +func Test_Notification(t *testing.T) { + alice := test.NewUser() + aliceLocalpart, _, err := gomatrixserverlib.SplitID('@', alice.ID) + assert.NoError(t, err) + room := test.NewRoom(t, alice) + room2 := test.NewRoom(t, alice) + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, close := mustCreateDatabase(t, dbType) + defer close() + // generate some dummy notifications + for i := 0; i < 10; i++ { + eventID := util.RandomString(16) + roomID := room.ID + ts := time.Now() + if i > 5 { + roomID = room2.ID + // create some old notifications to test DeleteOldNotifications + ts = ts.AddDate(0, -2, 0) + } + notification := &api.Notification{ + Actions: []*pushrules.Action{ + {}, + }, + Event: gomatrixserverlib.ClientEvent{ + Content: gomatrixserverlib.RawJSON("{}"), + }, + Read: false, + RoomID: roomID, + TS: gomatrixserverlib.AsTimestamp(ts), + } + err = db.InsertNotification(ctx, aliceLocalpart, eventID, int64(i+1), nil, notification) + assert.NoError(t, err, "unable to insert notification") + } + + // get notifications + count, err := db.GetNotificationCount(ctx, aliceLocalpart, tables.AllNotifications) + assert.NoError(t, err, "unable to get notification count") + assert.Equal(t, int64(10), count) + notifs, count, err := db.GetNotifications(ctx, aliceLocalpart, 0, 15, tables.AllNotifications) + assert.NoError(t, err, "unable to get notifications") + assert.Equal(t, int64(10), count) + assert.Equal(t, 10, len(notifs)) + // ... for a specific room + total, _, err := db.GetRoomNotificationCounts(ctx, aliceLocalpart, room2.ID) + assert.NoError(t, err, "unable to get notifications for room") + assert.Equal(t, int64(4), total) + + // mark notification as read + affected, err := db.SetNotificationsRead(ctx, aliceLocalpart, room2.ID, 7, true) + assert.NoError(t, err, "unable to set notifications read") + assert.True(t, affected) + + // this should delete 2 notifications + affected, err = db.DeleteNotificationsUpTo(ctx, aliceLocalpart, room2.ID, 8) + assert.NoError(t, err, "unable to set notifications read") + assert.True(t, affected) + + total, _, err = db.GetRoomNotificationCounts(ctx, aliceLocalpart, room2.ID) + assert.NoError(t, err, "unable to get notifications for room") + assert.Equal(t, int64(2), total) + + // delete old notifications + err = db.DeleteOldNotifications(ctx) + assert.NoError(t, err) + + // this should now return 0 notifications + total, _, err = db.GetRoomNotificationCounts(ctx, aliceLocalpart, room2.ID) + assert.NoError(t, err, "unable to get notifications for room") + assert.Equal(t, int64(0), total) + }) +} diff --git a/userapi/storage/storage_wasm.go b/userapi/storage/storage_wasm.go index 779f77568..a8e6f031c 100644 --- a/userapi/storage/storage_wasm.go +++ b/userapi/storage/storage_wasm.go @@ -23,7 +23,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -func NewDatabase( +func NewUserAPIDatabase( dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go index 8c3608bd8..076b4f3c6 100644 --- a/userapi/userapi_test.go +++ b/userapi/userapi_test.go @@ -52,7 +52,7 @@ func MustMakeInternalAPI(t *testing.T, opts apiTestOpts) (api.UserInternalAPI, s MaxOpenConnections: 1, MaxIdleConnections: 1, } - accountDB, err := storage.NewDatabase(dbopts, serverName, bcrypt.MinCost, config.DefaultOpenIDTokenLifetimeMS, opts.loginTokenLifetime, "") + accountDB, err := storage.NewUserAPIDatabase(dbopts, serverName, bcrypt.MinCost, config.DefaultOpenIDTokenLifetimeMS, opts.loginTokenLifetime, "") if err != nil { t.Fatalf("failed to create account DB: %s", err) } From 6ee8507955f2b9674649acc928768b1a4d96f7c0 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 27 Apr 2022 14:45:51 +0100 Subject: [PATCH 27/50] Correct account data position mapping --- syncapi/storage/postgres/account_data_table.go | 10 +++++++--- syncapi/storage/sqlite3/account_data_table.go | 11 +++++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/syncapi/storage/postgres/account_data_table.go b/syncapi/storage/postgres/account_data_table.go index ec1919fca..7c0d03030 100644 --- a/syncapi/storage/postgres/account_data_table.go +++ b/syncapi/storage/postgres/account_data_table.go @@ -105,7 +105,7 @@ func (s *accountDataStatements) SelectAccountDataInRange( accountDataEventFilter *gomatrixserverlib.EventFilter, ) (data map[string][]string, pos types.StreamPosition, err error) { data = make(map[string][]string) - pos = r.Low() + pos = r.High() rows, err := s.selectAccountDataInRangeStmt.QueryContext(ctx, userID, r.Low(), r.High(), pq.StringArray(filterConvertTypeWildcardToSQL(accountDataEventFilter.Types)), @@ -120,6 +120,7 @@ func (s *accountDataStatements) SelectAccountDataInRange( var dataType string var roomID string var id types.StreamPosition + var highest types.StreamPosition for rows.Next() { if err = rows.Scan(&id, &roomID, &dataType); err != nil { @@ -131,10 +132,13 @@ func (s *accountDataStatements) SelectAccountDataInRange( } else { data[roomID] = []string{dataType} } - if id > pos { - pos = id + if id > highest { + highest = id } } + if highest < pos { + pos = highest + } return data, pos, rows.Err() } diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go index 2c7272ea8..1bbfe9c96 100644 --- a/syncapi/storage/sqlite3/account_data_table.go +++ b/syncapi/storage/sqlite3/account_data_table.go @@ -96,7 +96,7 @@ func (s *accountDataStatements) SelectAccountDataInRange( r types.Range, filter *gomatrixserverlib.EventFilter, ) (data map[string][]string, pos types.StreamPosition, err error) { - pos = r.Low() + pos = r.High() data = make(map[string][]string) stmt, params, err := prepareWithFilters( s.db, nil, selectAccountDataInRangeSQL, @@ -116,6 +116,7 @@ func (s *accountDataStatements) SelectAccountDataInRange( var dataType string var roomID string var id types.StreamPosition + var highest types.StreamPosition for rows.Next() { if err = rows.Scan(&id, &roomID, &dataType); err != nil { @@ -127,11 +128,13 @@ func (s *accountDataStatements) SelectAccountDataInRange( } else { data[roomID] = []string{dataType} } - if id > pos { - pos = id + if id > highest { + highest = id } } - + if highest < pos { + pos = highest + } return data, pos, nil } From 655ac3e8fb83e1cb9b670ab420a0f661dc19786e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 27 Apr 2022 14:53:11 +0100 Subject: [PATCH 28/50] Try that again --- syncapi/storage/postgres/account_data_table.go | 10 ++++------ syncapi/storage/sqlite3/account_data_table.go | 10 ++++------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/syncapi/storage/postgres/account_data_table.go b/syncapi/storage/postgres/account_data_table.go index 7c0d03030..e9c72058b 100644 --- a/syncapi/storage/postgres/account_data_table.go +++ b/syncapi/storage/postgres/account_data_table.go @@ -105,7 +105,6 @@ func (s *accountDataStatements) SelectAccountDataInRange( accountDataEventFilter *gomatrixserverlib.EventFilter, ) (data map[string][]string, pos types.StreamPosition, err error) { data = make(map[string][]string) - pos = r.High() rows, err := s.selectAccountDataInRangeStmt.QueryContext(ctx, userID, r.Low(), r.High(), pq.StringArray(filterConvertTypeWildcardToSQL(accountDataEventFilter.Types)), @@ -120,7 +119,6 @@ func (s *accountDataStatements) SelectAccountDataInRange( var dataType string var roomID string var id types.StreamPosition - var highest types.StreamPosition for rows.Next() { if err = rows.Scan(&id, &roomID, &dataType); err != nil { @@ -132,12 +130,12 @@ func (s *accountDataStatements) SelectAccountDataInRange( } else { data[roomID] = []string{dataType} } - if id > highest { - highest = id + if id > pos { + pos = id } } - if highest < pos { - pos = highest + if pos == 0 { + pos = r.High() } return data, pos, rows.Err() } diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go index 1bbfe9c96..21a16dcd3 100644 --- a/syncapi/storage/sqlite3/account_data_table.go +++ b/syncapi/storage/sqlite3/account_data_table.go @@ -96,7 +96,6 @@ func (s *accountDataStatements) SelectAccountDataInRange( r types.Range, filter *gomatrixserverlib.EventFilter, ) (data map[string][]string, pos types.StreamPosition, err error) { - pos = r.High() data = make(map[string][]string) stmt, params, err := prepareWithFilters( s.db, nil, selectAccountDataInRangeSQL, @@ -116,7 +115,6 @@ func (s *accountDataStatements) SelectAccountDataInRange( var dataType string var roomID string var id types.StreamPosition - var highest types.StreamPosition for rows.Next() { if err = rows.Scan(&id, &roomID, &dataType); err != nil { @@ -128,12 +126,12 @@ func (s *accountDataStatements) SelectAccountDataInRange( } else { data[roomID] = []string{dataType} } - if id > highest { - highest = id + if id > pos { + pos = id } } - if highest < pos { - pos = highest + if pos == 0 { + pos = r.High() } return data, pos, nil } From cafa2853c5d67b3dd4d247abdd1ad5806f0c951b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 27 Apr 2022 15:01:57 +0100 Subject: [PATCH 29/50] Use process context as base context for all HTTP --- setup/base/base.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup/base/base.go b/setup/base/base.go index dbc5d2394..51c43198a 100644 --- a/setup/base/base.go +++ b/setup/base/base.go @@ -346,6 +346,9 @@ func (b *BaseDendrite) SetupAndServeHTTP( Addr: string(externalAddr), WriteTimeout: HTTPServerTimeout, Handler: externalRouter, + BaseContext: func(_ net.Listener) context.Context { + return b.ProcessContext.Context() + }, } internalServ := externalServ @@ -361,6 +364,9 @@ func (b *BaseDendrite) SetupAndServeHTTP( internalServ = &http.Server{ Addr: string(internalAddr), Handler: h2c.NewHandler(internalRouter, internalH2S), + BaseContext: func(_ net.Listener) context.Context { + return b.ProcessContext.Context() + }, } } From 103795d33a09728d7619e73014d507505ff121e2 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 27 Apr 2022 15:06:20 +0100 Subject: [PATCH 30/50] Defer cancel on shutdown context --- setup/base/base.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/base/base.go b/setup/base/base.go index 51c43198a..03ea2ad7e 100644 --- a/setup/base/base.go +++ b/setup/base/base.go @@ -472,7 +472,7 @@ func (b *BaseDendrite) SetupAndServeHTTP( b.WaitForShutdown() ctx, cancel := context.WithCancel(context.Background()) - cancel() + defer cancel() _ = internalServ.Shutdown(ctx) _ = externalServ.Shutdown(ctx) From 923f789ca3174a685bd53ce5e64a5e86cabd38cb Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 27 Apr 2022 15:29:49 +0100 Subject: [PATCH 31/50] Fix graceful shutdown --- federationapi/queue/destinationqueue.go | 13 ++++++++----- federationapi/queue/queue.go | 13 ++++++------- setup/base/base.go | 12 ++++++------ setup/jetstream/helpers.go | 16 +++++++++++++--- 4 files changed, 33 insertions(+), 21 deletions(-) diff --git a/federationapi/queue/destinationqueue.go b/federationapi/queue/destinationqueue.go index a5f8c03b9..747940403 100644 --- a/federationapi/queue/destinationqueue.go +++ b/federationapi/queue/destinationqueue.go @@ -78,7 +78,7 @@ func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, re // this destination queue. We'll then be able to retrieve the PDU // later. if err := oq.db.AssociatePDUWithDestination( - context.TODO(), + oq.process.Context(), "", // TODO: remove this, as we don't need to persist the transaction ID oq.destination, // the destination server name receipt, // NIDs from federationapi_queue_json table @@ -122,7 +122,7 @@ func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, receipt *share // this destination queue. We'll then be able to retrieve the PDU // later. if err := oq.db.AssociateEDUWithDestination( - context.TODO(), + oq.process.Context(), oq.destination, // the destination server name receipt, // NIDs from federationapi_queue_json table event.Type, @@ -177,7 +177,7 @@ func (oq *destinationQueue) getPendingFromDatabase() { // Check to see if there's anything to do for this server // in the database. retrieved := false - ctx := context.Background() + ctx := oq.process.Context() oq.pendingMutex.Lock() defer oq.pendingMutex.Unlock() @@ -271,6 +271,9 @@ func (oq *destinationQueue) backgroundSend() { // restarted automatically the next time we have an event to // send. return + case <-oq.process.Context().Done(): + // The parent process is shutting down, so stop. + return } // If we are backing off this server then wait for the @@ -420,13 +423,13 @@ func (oq *destinationQueue) nextTransaction( // Clean up the transaction in the database. if pduReceipts != nil { //logrus.Infof("Cleaning PDUs %q", pduReceipt.String()) - if err = oq.db.CleanPDUs(context.Background(), oq.destination, pduReceipts); err != nil { + if err = oq.db.CleanPDUs(oq.process.Context(), oq.destination, pduReceipts); err != nil { logrus.WithError(err).Errorf("Failed to clean PDUs for server %q", t.Destination) } } if eduReceipts != nil { //logrus.Infof("Cleaning EDUs %q", eduReceipt.String()) - if err = oq.db.CleanEDUs(context.Background(), oq.destination, eduReceipts); err != nil { + if err = oq.db.CleanEDUs(oq.process.Context(), oq.destination, eduReceipts); err != nil { logrus.WithError(err).Errorf("Failed to clean EDUs for server %q", t.Destination) } } diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go index c45bbd1d4..d152886f5 100644 --- a/federationapi/queue/queue.go +++ b/federationapi/queue/queue.go @@ -15,7 +15,6 @@ package queue import ( - "context" "crypto/ed25519" "encoding/json" "fmt" @@ -105,14 +104,14 @@ func NewOutgoingQueues( // Look up which servers we have pending items for and then rehydrate those queues. if !disabled { serverNames := map[gomatrixserverlib.ServerName]struct{}{} - if names, err := db.GetPendingPDUServerNames(context.Background()); err == nil { + if names, err := db.GetPendingPDUServerNames(process.Context()); err == nil { for _, serverName := range names { serverNames[serverName] = struct{}{} } } else { log.WithError(err).Error("Failed to get PDU server names for destination queue hydration") } - if names, err := db.GetPendingEDUServerNames(context.Background()); err == nil { + if names, err := db.GetPendingEDUServerNames(process.Context()); err == nil { for _, serverName := range names { serverNames[serverName] = struct{}{} } @@ -215,7 +214,7 @@ func (oqs *OutgoingQueues) SendEvent( // Check if any of the destinations are prohibited by server ACLs. for destination := range destmap { if api.IsServerBannedFromRoom( - context.TODO(), + oqs.process.Context(), oqs.rsAPI, ev.RoomID(), destination, @@ -238,7 +237,7 @@ func (oqs *OutgoingQueues) SendEvent( return fmt.Errorf("json.Marshal: %w", err) } - nid, err := oqs.db.StoreJSON(context.TODO(), string(headeredJSON)) + nid, err := oqs.db.StoreJSON(oqs.process.Context(), string(headeredJSON)) if err != nil { return fmt.Errorf("sendevent: oqs.db.StoreJSON: %w", err) } @@ -286,7 +285,7 @@ func (oqs *OutgoingQueues) SendEDU( if result := gjson.GetBytes(e.Content, "room_id"); result.Exists() { for destination := range destmap { if api.IsServerBannedFromRoom( - context.TODO(), + oqs.process.Context(), oqs.rsAPI, result.Str, destination, @@ -310,7 +309,7 @@ func (oqs *OutgoingQueues) SendEDU( return fmt.Errorf("json.Marshal: %w", err) } - nid, err := oqs.db.StoreJSON(context.TODO(), string(ephemeralJSON)) + nid, err := oqs.db.StoreJSON(oqs.process.Context(), string(ephemeralJSON)) if err != nil { return fmt.Errorf("sendevent: oqs.db.StoreJSON: %w", err) } diff --git a/setup/base/base.go b/setup/base/base.go index 03ea2ad7e..e67b034a3 100644 --- a/setup/base/base.go +++ b/setup/base/base.go @@ -469,14 +469,14 @@ func (b *BaseDendrite) SetupAndServeHTTP( } minwinsvc.SetOnExit(b.ProcessContext.ShutdownDendrite) - b.WaitForShutdown() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - _ = internalServ.Shutdown(ctx) - _ = externalServ.Shutdown(ctx) + <-b.ProcessContext.WaitForShutdown() + logrus.Infof("Stopping HTTP listeners") + _ = internalServ.Shutdown(context.Background()) + _ = externalServ.Shutdown(context.Background()) logrus.Infof("Stopped HTTP listeners") + + b.WaitForShutdown() } func (b *BaseDendrite) WaitForShutdown() { diff --git a/setup/jetstream/helpers.go b/setup/jetstream/helpers.go index 78cecb6ae..1c07583e9 100644 --- a/setup/jetstream/helpers.go +++ b/setup/jetstream/helpers.go @@ -35,6 +35,16 @@ func JetStreamConsumer( } go func() { for { + // If the parent context has given up then there's no point in + // carrying on doing anything, so stop the listener. + select { + case <-ctx.Done(): + if err := sub.Unsubscribe(); err != nil { + logrus.WithContext(ctx).Warnf("Failed to unsubscribe %q", durable) + } + return + default: + } // The context behaviour here is surprising — we supply a context // so that we can interrupt the fetch if we want, but NATS will still // enforce its own deadline (roughly 5 seconds by default). Therefore @@ -65,18 +75,18 @@ func JetStreamConsumer( continue } msg := msgs[0] - if err = msg.InProgress(); err != nil { + if err = msg.InProgress(nats.Context(ctx)); err != nil { logrus.WithContext(ctx).WithField("subject", subj).Warn(fmt.Errorf("msg.InProgress: %w", err)) sentry.CaptureException(err) continue } if f(ctx, msg) { - if err = msg.AckSync(); err != nil { + if err = msg.AckSync(nats.Context(ctx)); err != nil { logrus.WithContext(ctx).WithField("subject", subj).Warn(fmt.Errorf("msg.AckSync: %w", err)) sentry.CaptureException(err) } } else { - if err = msg.Nak(); err != nil { + if err = msg.Nak(nats.Context(ctx)); err != nil { logrus.WithContext(ctx).WithField("subject", subj).Warn(fmt.Errorf("msg.Nak: %w", err)) sentry.CaptureException(err) } From 34221938ccb1f3a885ac9e5a36b79d3d74850d38 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 27 Apr 2022 16:04:11 +0100 Subject: [PATCH 32/50] Version 0.8.2 (#2386) * Version 0.8.2 * Correct account data position mapping * Try that again * Don't duplicate wait-for-shutdowns --- CHANGES.md | 29 +++++++++++++++++++++++++++++ internal/version.go | 2 +- setup/base/base.go | 4 +--- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 831a8969d..6278bcba4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,34 @@ # Changelog +## Dendrite 0.8.2 (2022-04-27) + +### Features + +* Lazy-loading has been added to the `/sync` endpoint, which should speed up syncs considerably +* Filtering has been added to the `/messages` endpoint +* The room summary now contains "heroes" (up to 5 users in the room) for clients to display when no room name is set +* The existing lazy-loading caches will now be used by `/messages` and `/context` so that member events will not be sent to clients more times than necessary +* The account data stream now uses the provided filters +* The built-in NATS Server has been updated to version 2.8.0 +* The `/state` and `/state_ids` endpoints will now return `M_NOT_FOUND` for rejected events +* Repeated calls to the `/redact` endpoint will now be idempotent when a transaction ID is given +* Dendrite should now be able to run as a Windows service under Service Control Manager + +### Fixes + +* Fictitious presence updates will no longer be created for users which have not sent us presence updates, which should speed up complete syncs considerably +* Uploading cross-signing device signatures should now be more reliable, fixing a number of bugs with cross-signing +* All account data should now be sent properly on a complete sync, which should eliminate problems with client settings or key backups appearing to be missing +* Account data will now be limited correctly on incremental syncs, returning the stream position of the most recent update rather than the latest stream position +* Account data will not be sent for parted rooms, which should reduce the number of left/forgotten rooms reappearing in clients as empty rooms +* The TURN username hash has been fixed which should help to resolve some problems when using TURN for voice calls (contributed by [fcwoknhenuxdfiyv](https://github.com/fcwoknhenuxdfiyv)) +* Push rules can no longer be modified using the account data endpoints +* Querying account availability should now work properly in polylith deployments +* A number of bugs with sync filters have been fixed +* A default sync filter will now be used if the request contains a filter ID that does not exist +* The `pushkey_ts` field is now using seconds instead of milliseconds +* A race condition when gracefully shutting down has been fixed, so JetStream should no longer cause the process to exit before other Dendrite components are finished shutting down + ## Dendrite 0.8.1 (2022-04-07) ### Fixes diff --git a/internal/version.go b/internal/version.go index 5227a03bf..2477bc9ac 100644 --- a/internal/version.go +++ b/internal/version.go @@ -17,7 +17,7 @@ var build string const ( VersionMajor = 0 VersionMinor = 8 - VersionPatch = 1 + VersionPatch = 2 VersionTag = "" // example: "rc1" ) diff --git a/setup/base/base.go b/setup/base/base.go index e67b034a3..7091c6ba5 100644 --- a/setup/base/base.go +++ b/setup/base/base.go @@ -469,14 +469,12 @@ func (b *BaseDendrite) SetupAndServeHTTP( } minwinsvc.SetOnExit(b.ProcessContext.ShutdownDendrite) - <-b.ProcessContext.WaitForShutdown() + logrus.Infof("Stopping HTTP listeners") _ = internalServ.Shutdown(context.Background()) _ = externalServ.Shutdown(context.Background()) logrus.Infof("Stopped HTTP listeners") - - b.WaitForShutdown() } func (b *BaseDendrite) WaitForShutdown() { From 8d69e2f0b897dd1ecd99c7bd348ad8bfc8999c4e Mon Sep 17 00:00:00 2001 From: 0x1a8510f2 Date: Wed, 27 Apr 2022 20:19:46 +0100 Subject: [PATCH 33/50] Use Go 1.18 to build Docker images (#2391) Go 1.18 has now been released for a while and the CI already tests Dendrite with Go 1.18 so there should be no issues. Go 1.18 brings some performance improvements for ARM via the register calling convention so it makes sense to switch to it. --- build/docker/Dockerfile.monolith | 4 ++-- build/docker/Dockerfile.polylith | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build/docker/Dockerfile.monolith b/build/docker/Dockerfile.monolith index 0d2a141ad..891a3a9e0 100644 --- a/build/docker/Dockerfile.monolith +++ b/build/docker/Dockerfile.monolith @@ -1,4 +1,4 @@ -FROM docker.io/golang:1.17-alpine AS base +FROM docker.io/golang:1.18-alpine AS base RUN apk --update --no-cache add bash build-base @@ -23,4 +23,4 @@ COPY --from=base /build/bin/* /usr/bin/ VOLUME /etc/dendrite WORKDIR /etc/dendrite -ENTRYPOINT ["/usr/bin/dendrite-monolith-server"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/dendrite-monolith-server"] diff --git a/build/docker/Dockerfile.polylith b/build/docker/Dockerfile.polylith index c266fd480..ffdc35586 100644 --- a/build/docker/Dockerfile.polylith +++ b/build/docker/Dockerfile.polylith @@ -1,4 +1,4 @@ -FROM docker.io/golang:1.17-alpine AS base +FROM docker.io/golang:1.18-alpine AS base RUN apk --update --no-cache add bash build-base @@ -23,4 +23,4 @@ COPY --from=base /build/bin/* /usr/bin/ VOLUME /etc/dendrite WORKDIR /etc/dendrite -ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"] From 74259f296f225510e9fbb6c5aae191c3f86c729e Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 27 Apr 2022 21:31:30 +0200 Subject: [PATCH 34/50] Fix #2390 (#2392) Fix duplicate heroes in `/sync` response. --- syncapi/storage/postgres/memberships_table.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncapi/storage/postgres/memberships_table.go b/syncapi/storage/postgres/memberships_table.go index 8c049977f..00223c57a 100644 --- a/syncapi/storage/postgres/memberships_table.go +++ b/syncapi/storage/postgres/memberships_table.go @@ -64,7 +64,7 @@ const selectMembershipCountSQL = "" + ") t WHERE t.membership = $3" const selectHeroesSQL = "" + - "SELECT user_id FROM syncapi_memberships WHERE room_id = $1 AND user_id != $2 AND membership = ANY($3) LIMIT 5" + "SELECT DISTINCT user_id FROM syncapi_memberships WHERE room_id = $1 AND user_id != $2 AND membership = ANY($3) LIMIT 5" type membershipsStatements struct { upsertMembershipStmt *sql.Stmt From 2ff75b7c806829b211be0c310497728055e898cc Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 28 Apr 2022 11:34:19 +0100 Subject: [PATCH 35/50] Ensure signature map exists (fixes #2393) (#2397) --- keyserver/internal/internal.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index e571c7e56..1677cf8e3 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -319,6 +319,9 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques // JSON, add the signatures and marshal it again, for some reason? for targetUserID, masterKey := range res.MasterKeys { + if masterKey.Signatures == nil { + masterKey.Signatures = map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{} + } for targetKeyID := range masterKey.Keys { sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, req.UserID, targetUserID, targetKeyID) if err != nil { From 6deb10f3f61e4da0360a8910f6b5375a2d31f182 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 28 Apr 2022 11:45:56 +0100 Subject: [PATCH 36/50] Don't answer expensive federation requests for rooms we no longer belong to (#2398) This includes `/state`, `/state_ids`, `/get_missing_events` and `/backfill`. This should fix #2396. --- federationapi/routing/backfill.go | 6 ++++++ federationapi/routing/eventauth.go | 6 ++++++ federationapi/routing/missingevents.go | 6 ++++++ federationapi/routing/routing.go | 27 ++++++++++++++++++++++++++ federationapi/routing/state.go | 6 ++++++ 5 files changed, 51 insertions(+) diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index 31005209f..82f6cbabf 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -51,6 +51,12 @@ func Backfill( } } + // If we don't think we belong to this room then don't waste the effort + // responding to expensive requests for it. + if err := ErrorIfLocalServerNotInRoom(httpReq.Context(), rsAPI, roomID); err != nil { + return *err + } + // Check if all of the required parameters are there. eIDs, exists = httpReq.URL.Query()["v"] if !exists { diff --git a/federationapi/routing/eventauth.go b/federationapi/routing/eventauth.go index 0a03a0cb4..e83cb8ad2 100644 --- a/federationapi/routing/eventauth.go +++ b/federationapi/routing/eventauth.go @@ -30,6 +30,12 @@ func GetEventAuth( roomID string, eventID string, ) util.JSONResponse { + // If we don't think we belong to this room then don't waste the effort + // responding to expensive requests for it. + if err := ErrorIfLocalServerNotInRoom(ctx, rsAPI, roomID); err != nil { + return *err + } + event, resErr := fetchEvent(ctx, rsAPI, eventID) if resErr != nil { return *resErr diff --git a/federationapi/routing/missingevents.go b/federationapi/routing/missingevents.go index dd3df7aa9..b826d69c4 100644 --- a/federationapi/routing/missingevents.go +++ b/federationapi/routing/missingevents.go @@ -45,6 +45,12 @@ func GetMissingEvents( } } + // If we don't think we belong to this room then don't waste the effort + // responding to expensive requests for it. + if err := ErrorIfLocalServerNotInRoom(httpReq.Context(), rsAPI, roomID); err != nil { + return *err + } + var eventsResponse api.QueryMissingEventsResponse if err := rsAPI.QueryMissingEvents( httpReq.Context(), &api.QueryMissingEventsRequest{ diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index a085ed780..6d24c8b40 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -15,6 +15,8 @@ package routing import ( + "context" + "fmt" "net/http" "github.com/gorilla/mux" @@ -24,6 +26,7 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/httputil" keyserverAPI "github.com/matrix-org/dendrite/keyserver/api" + "github.com/matrix-org/dendrite/roomserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -491,3 +494,27 @@ func Setup( }), ).Methods(http.MethodGet) } + +func ErrorIfLocalServerNotInRoom( + ctx context.Context, + rsAPI api.RoomserverInternalAPI, + roomID string, +) *util.JSONResponse { + // Check if we think we're in this room. If we aren't then + // we won't waste CPU cycles serving this request. + joinedReq := &api.QueryServerJoinedToRoomRequest{ + RoomID: roomID, + } + joinedRes := &api.QueryServerJoinedToRoomResponse{} + if err := rsAPI.QueryServerJoinedToRoom(ctx, joinedReq, joinedRes); err != nil { + res := util.ErrorResponse(err) + return &res + } + if !joinedRes.IsInRoom { + return &util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound(fmt.Sprintf("This server is not joined to room %s", roomID)), + } + } + return nil +} diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go index a202c92c2..e2b67776a 100644 --- a/federationapi/routing/state.go +++ b/federationapi/routing/state.go @@ -101,6 +101,12 @@ func getState( roomID string, eventID string, ) (stateEvents, authEvents []*gomatrixserverlib.HeaderedEvent, errRes *util.JSONResponse) { + // If we don't think we belong to this room then don't waste the effort + // responding to expensive requests for it. + if err := ErrorIfLocalServerNotInRoom(ctx, rsAPI, roomID); err != nil { + return nil, nil, err + } + event, resErr := fetchEvent(ctx, rsAPI, eventID) if resErr != nil { return nil, nil, resErr From 65034d1f227de45e88d39ec5a3e83d854e840875 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 28 Apr 2022 11:46:15 +0100 Subject: [PATCH 37/50] Unlist test since it no longer seems to be flakey (hopefully?) --- sytest-blacklist | 1 - 1 file changed, 1 deletion(-) diff --git a/sytest-blacklist b/sytest-blacklist index 713a5b631..be0826eee 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -48,4 +48,3 @@ Notifications can be viewed with GET /notifications # More flakey If remote user leaves room we no longer receive device updates -Local device key changes get to remote servers From 8683ff78b1bee6b7c35e7befb9903c794a17510c Mon Sep 17 00:00:00 2001 From: Till Faelligen Date: Thu, 28 Apr 2022 15:06:34 +0200 Subject: [PATCH 38/50] Make tests more reliable --- userapi/storage/postgres/devices_table.go | 13 ++++++++++--- userapi/storage/sqlite3/devices_table.go | 13 ++++++++++--- userapi/storage/storage_test.go | 8 ++++---- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/userapi/storage/postgres/devices_table.go b/userapi/storage/postgres/devices_table.go index fe8c54e04..6c777982f 100644 --- a/userapi/storage/postgres/devices_table.go +++ b/userapi/storage/postgres/devices_table.go @@ -75,7 +75,7 @@ const selectDeviceByTokenSQL = "" + "SELECT session_id, device_id, localpart FROM device_devices WHERE access_token = $1" const selectDeviceByIDSQL = "" + - "SELECT display_name FROM device_devices WHERE localpart = $1 and device_id = $2" + "SELECT display_name, last_seen_ts, ip FROM device_devices WHERE localpart = $1 and device_id = $2" const selectDevicesByLocalpartSQL = "" + "SELECT device_id, display_name, last_seen_ts, ip, user_agent FROM device_devices WHERE localpart = $1 AND device_id != $2 ORDER BY last_seen_ts DESC" @@ -215,15 +215,22 @@ func (s *devicesStatements) SelectDeviceByID( ctx context.Context, localpart, deviceID string, ) (*api.Device, error) { var dev api.Device - var displayName sql.NullString + var displayName, ip sql.NullString + var lastseenTS sql.NullInt64 stmt := s.selectDeviceByIDStmt - err := stmt.QueryRowContext(ctx, localpart, deviceID).Scan(&displayName) + err := stmt.QueryRowContext(ctx, localpart, deviceID).Scan(&displayName, &lastseenTS, &ip) if err == nil { dev.ID = deviceID dev.UserID = userutil.MakeUserID(localpart, s.serverName) if displayName.Valid { dev.DisplayName = displayName.String } + if lastseenTS.Valid { + dev.LastSeenTS = lastseenTS.Int64 + } + if ip.Valid { + dev.LastSeenIP = ip.String + } } return &dev, err } diff --git a/userapi/storage/sqlite3/devices_table.go b/userapi/storage/sqlite3/devices_table.go index 7860bd6a2..b86ed1cc2 100644 --- a/userapi/storage/sqlite3/devices_table.go +++ b/userapi/storage/sqlite3/devices_table.go @@ -60,7 +60,7 @@ const selectDeviceByTokenSQL = "" + "SELECT session_id, device_id, localpart FROM device_devices WHERE access_token = $1" const selectDeviceByIDSQL = "" + - "SELECT display_name FROM device_devices WHERE localpart = $1 and device_id = $2" + "SELECT display_name, last_seen_ts, ip FROM device_devices WHERE localpart = $1 and device_id = $2" const selectDevicesByLocalpartSQL = "" + "SELECT device_id, display_name, last_seen_ts, ip, user_agent FROM device_devices WHERE localpart = $1 AND device_id != $2 ORDER BY last_seen_ts DESC" @@ -212,15 +212,22 @@ func (s *devicesStatements) SelectDeviceByID( ctx context.Context, localpart, deviceID string, ) (*api.Device, error) { var dev api.Device - var displayName sql.NullString + var displayName, ip sql.NullString stmt := s.selectDeviceByIDStmt - err := stmt.QueryRowContext(ctx, localpart, deviceID).Scan(&displayName) + var lastseenTS sql.NullInt64 + err := stmt.QueryRowContext(ctx, localpart, deviceID).Scan(&displayName, &lastseenTS, &ip) if err == nil { dev.ID = deviceID dev.UserID = userutil.MakeUserID(localpart, s.serverName) if displayName.Valid { dev.DisplayName = displayName.String } + if lastseenTS.Valid { + dev.LastSeenTS = lastseenTS.Int64 + } + if ip.Valid { + dev.LastSeenIP = ip.String + } } return &dev, err } diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go index e6c7d35fc..2eb57d0bc 100644 --- a/userapi/storage/storage_test.go +++ b/userapi/storage/storage_test.go @@ -180,12 +180,12 @@ func Test_Devices(t *testing.T) { deviceWithID.DisplayName = newName deviceWithID.LastSeenIP = "127.0.0.1" deviceWithID.LastSeenTS = int64(gomatrixserverlib.AsTimestamp(time.Now().Truncate(time.Second))) - devices, err = db.GetDevicesByLocalpart(ctx, localpart) + gotDevice, err = db.GetDeviceByID(ctx, localpart, deviceWithID.ID) assert.NoError(t, err, "unable to get device by id") assert.Equal(t, 2, len(devices)) - assert.Equal(t, deviceWithID.DisplayName, devices[0].DisplayName) - assert.Equal(t, deviceWithID.LastSeenIP, devices[0].LastSeenIP) - truncatedTime := gomatrixserverlib.Timestamp(devices[0].LastSeenTS).Time().Truncate(time.Second) + assert.Equal(t, deviceWithID.DisplayName, gotDevice.DisplayName) + assert.Equal(t, deviceWithID.LastSeenIP, gotDevice.LastSeenIP) + truncatedTime := gomatrixserverlib.Timestamp(gotDevice.LastSeenTS).Time().Truncate(time.Second) assert.Equal(t, gomatrixserverlib.Timestamp(deviceWithID.LastSeenTS), gomatrixserverlib.AsTimestamp(truncatedTime)) // create one more device and remove the devices step by step From 21ee5b36a41f2cb3960f63ef6f19106d36312aae Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Thu, 28 Apr 2022 16:12:40 +0200 Subject: [PATCH 39/50] Limit presence in `/sync` responses (#2394) * Use filter and limit presence count * More limiting * More limiting * Fix unit test * Also limit presence by last_active_ts * Update query, use "from" as the initial lastPos * Get 1000 presence events, they are filtered later Co-authored-by: Neil Alexander --- syncapi/storage/interface.go | 2 +- syncapi/storage/postgres/presence_table.go | 9 ++++++--- syncapi/storage/shared/syncserver.go | 4 ++-- syncapi/storage/sqlite3/presence_table.go | 10 ++++++---- syncapi/storage/tables/interface.go | 2 +- syncapi/streams/stream_presence.go | 12 ++++++++++-- syncapi/sync/requestpool_test.go | 2 +- 7 files changed, 27 insertions(+), 14 deletions(-) diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 43aaa3588..486978598 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -159,6 +159,6 @@ type Database interface { type Presence interface { UpdatePresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (types.StreamPosition, error) GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error) - PresenceAfter(ctx context.Context, after types.StreamPosition) (map[string]*types.PresenceInternal, error) + PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error) } diff --git a/syncapi/storage/postgres/presence_table.go b/syncapi/storage/postgres/presence_table.go index 9f1e37f79..7194afea6 100644 --- a/syncapi/storage/postgres/presence_table.go +++ b/syncapi/storage/postgres/presence_table.go @@ -17,6 +17,7 @@ package postgres import ( "context" "database/sql" + "time" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" @@ -72,7 +73,8 @@ const selectMaxPresenceSQL = "" + const selectPresenceAfter = "" + " SELECT id, user_id, presence, status_msg, last_active_ts" + " FROM syncapi_presence" + - " WHERE id > $1" + " WHERE id > $1 AND last_active_ts >= $2" + + " ORDER BY id ASC LIMIT $3" type presenceStatements struct { upsertPresenceStmt *sql.Stmt @@ -144,11 +146,12 @@ func (p *presenceStatements) GetMaxPresenceID(ctx context.Context, txn *sql.Tx) func (p *presenceStatements) GetPresenceAfter( ctx context.Context, txn *sql.Tx, after types.StreamPosition, + filter gomatrixserverlib.EventFilter, ) (presences map[string]*types.PresenceInternal, err error) { presences = make(map[string]*types.PresenceInternal) stmt := sqlutil.TxStmt(txn, p.selectPresenceAfterStmt) - - rows, err := stmt.QueryContext(ctx, after) + afterTS := gomatrixserverlib.AsTimestamp(time.Now().Add(time.Minute * -5)) + rows, err := stmt.QueryContext(ctx, after, afterTS, filter.Limit) if err != nil { return nil, err } diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index 25aca50ae..b7d2d3a29 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -1056,8 +1056,8 @@ func (s *Database) GetPresence(ctx context.Context, userID string) (*types.Prese return s.Presence.GetPresenceForUser(ctx, nil, userID) } -func (s *Database) PresenceAfter(ctx context.Context, after types.StreamPosition) (map[string]*types.PresenceInternal, error) { - return s.Presence.GetPresenceAfter(ctx, nil, after) +func (s *Database) PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) { + return s.Presence.GetPresenceAfter(ctx, nil, after, filter) } func (s *Database) MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error) { diff --git a/syncapi/storage/sqlite3/presence_table.go b/syncapi/storage/sqlite3/presence_table.go index 177a01bf3..b61a825df 100644 --- a/syncapi/storage/sqlite3/presence_table.go +++ b/syncapi/storage/sqlite3/presence_table.go @@ -17,6 +17,7 @@ package sqlite3 import ( "context" "database/sql" + "time" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" @@ -71,7 +72,8 @@ const selectMaxPresenceSQL = "" + const selectPresenceAfter = "" + " SELECT id, user_id, presence, status_msg, last_active_ts" + " FROM syncapi_presence" + - " WHERE id > $1" + " WHERE id > $1 AND last_active_ts >= $2" + + " ORDER BY id ASC LIMIT $3" type presenceStatements struct { db *sql.DB @@ -158,12 +160,12 @@ func (p *presenceStatements) GetMaxPresenceID(ctx context.Context, txn *sql.Tx) // GetPresenceAfter returns the changes presences after a given stream id func (p *presenceStatements) GetPresenceAfter( ctx context.Context, txn *sql.Tx, - after types.StreamPosition, + after types.StreamPosition, filter gomatrixserverlib.EventFilter, ) (presences map[string]*types.PresenceInternal, err error) { presences = make(map[string]*types.PresenceInternal) stmt := sqlutil.TxStmt(txn, p.selectPresenceAfterStmt) - - rows, err := stmt.QueryContext(ctx, after) + afterTS := gomatrixserverlib.AsTimestamp(time.Now().Add(time.Minute * -5)) + rows, err := stmt.QueryContext(ctx, after, afterTS, filter.Limit) if err != nil { return nil, err } diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index 4ff4689ed..6fdd9483e 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -188,5 +188,5 @@ type Presence interface { UpsertPresence(ctx context.Context, txn *sql.Tx, userID string, statusMsg *string, presence types.Presence, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (pos types.StreamPosition, err error) GetPresenceForUser(ctx context.Context, txn *sql.Tx, userID string) (presence *types.PresenceInternal, err error) GetMaxPresenceID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) - GetPresenceAfter(ctx context.Context, txn *sql.Tx, after types.StreamPosition) (presences map[string]*types.PresenceInternal, err error) + GetPresenceAfter(ctx context.Context, txn *sql.Tx, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (presences map[string]*types.PresenceInternal, err error) } diff --git a/syncapi/streams/stream_presence.go b/syncapi/streams/stream_presence.go index 614b88d48..675a7a178 100644 --- a/syncapi/streams/stream_presence.go +++ b/syncapi/streams/stream_presence.go @@ -53,7 +53,8 @@ func (p *PresenceStreamProvider) IncrementalSync( req *types.SyncRequest, from, to types.StreamPosition, ) types.StreamPosition { - presences, err := p.DB.PresenceAfter(ctx, from) + // We pull out a larger number than the filter asks for, since we're filtering out events later + presences, err := p.DB.PresenceAfter(ctx, from, gomatrixserverlib.EventFilter{Limit: 1000}) if err != nil { req.Log.WithError(err).Error("p.DB.PresenceAfter failed") return from @@ -72,6 +73,7 @@ func (p *PresenceStreamProvider) IncrementalSync( req.Log.WithError(err).Error("unable to refresh notifier lists") return from } + NewlyJoinedLoop: for _, roomID := range newlyJoined { roomUsers := p.notifier.JoinedUsers(roomID) for i := range roomUsers { @@ -86,11 +88,14 @@ func (p *PresenceStreamProvider) IncrementalSync( req.Log.WithError(err).Error("unable to query presence for user") return from } + if len(presences) > req.Filter.Presence.Limit { + break NewlyJoinedLoop + } } } } - lastPos := to + lastPos := from for _, presence := range presences { if presence == nil { continue @@ -135,6 +140,9 @@ func (p *PresenceStreamProvider) IncrementalSync( if presence.StreamPos > lastPos { lastPos = presence.StreamPos } + if len(req.Response.Presence.Events) == req.Filter.Presence.Limit { + break + } p.cache.Store(cacheKey, presence) } diff --git a/syncapi/sync/requestpool_test.go b/syncapi/sync/requestpool_test.go index a80089945..5e52bc7c9 100644 --- a/syncapi/sync/requestpool_test.go +++ b/syncapi/sync/requestpool_test.go @@ -30,7 +30,7 @@ func (d dummyDB) GetPresence(ctx context.Context, userID string) (*types.Presenc return &types.PresenceInternal{}, nil } -func (d dummyDB) PresenceAfter(ctx context.Context, after types.StreamPosition) (map[string]*types.PresenceInternal, error) { +func (d dummyDB) PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) { return map[string]*types.PresenceInternal{}, nil } From c6ea2c9ff26ca6ae4c799db08a3f72c6b4d99256 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 28 Apr 2022 16:02:30 +0100 Subject: [PATCH 40/50] Add `/_dendrite/admin/evacuateRoom/{roomID}` (#2401) * Add new endpoint to allow admins to evacuate the local server from the room * Guard endpoint * Use right prefix * Auth API * More useful return error rather than a panic * More useful return value again * Update the path * Try using inputer instead * oh provide the config * Try that again * Return affected user IDs * Don't create so many forward extremities * Add missing `Path` to name Co-authored-by: Till <2353100+S7evinK@users.noreply.github.com> --- build/gobind-pinecone/monolith.go | 1 + build/gobind-yggdrasil/monolith.go | 1 + clientapi/clientapi.go | 4 +- clientapi/routing/routing.go | 42 ++++- cmd/dendrite-demo-pinecone/main.go | 1 + cmd/dendrite-demo-yggdrasil/main.go | 1 + cmd/dendrite-monolith-server/main.go | 1 + .../personalities/clientapi.go | 6 +- cmd/dendritejs-pinecone/main.go | 1 + roomserver/api/api.go | 6 + roomserver/api/api_trace.go | 9 + roomserver/api/perform.go | 9 + roomserver/internal/api.go | 7 + roomserver/internal/perform/perform_admin.go | 162 ++++++++++++++++++ roomserver/inthttp/client.go | 38 ++-- roomserver/inthttp/server.go | 11 ++ setup/monolith.go | 4 +- 17 files changed, 288 insertions(+), 16 deletions(-) create mode 100644 roomserver/internal/perform/perform_admin.go diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 9cc94d650..6b2533491 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -314,6 +314,7 @@ func (m *DendriteMonolith) Start() { base.PublicWellKnownAPIMux, base.PublicMediaAPIMux, base.SynapseAdminMux, + base.DendriteAdminMux, ) httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go index 87dcad2e8..b9c6c1b78 100644 --- a/build/gobind-yggdrasil/monolith.go +++ b/build/gobind-yggdrasil/monolith.go @@ -152,6 +152,7 @@ func (m *DendriteMonolith) Start() { base.PublicWellKnownAPIMux, base.PublicMediaAPIMux, base.SynapseAdminMux, + base.DendriteAdminMux, ) httpRouter := mux.NewRouter() diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index e2f8d3f32..ad277056c 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -36,6 +36,7 @@ func AddPublicRoutes( process *process.ProcessContext, router *mux.Router, synapseAdminRouter *mux.Router, + dendriteAdminRouter *mux.Router, cfg *config.ClientAPI, federation *gomatrixserverlib.FederationClient, rsAPI roomserverAPI.RoomserverInternalAPI, @@ -62,7 +63,8 @@ func AddPublicRoutes( } routing.Setup( - router, synapseAdminRouter, cfg, rsAPI, asAPI, + router, synapseAdminRouter, dendriteAdminRouter, + cfg, rsAPI, asAPI, userAPI, userDirectoryProvider, federation, syncProducer, transactionsCache, fsAPI, keyAPI, extRoomsProvider, mscCfg, natsClient, diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index f370b4f8c..ec90b80db 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -48,7 +48,8 @@ import ( // applied: // nolint: gocyclo func Setup( - publicAPIMux, synapseAdminRouter *mux.Router, cfg *config.ClientAPI, + publicAPIMux, synapseAdminRouter, dendriteAdminRouter *mux.Router, + cfg *config.ClientAPI, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, userAPI userapi.UserInternalAPI, @@ -119,6 +120,45 @@ func Setup( ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) } + dendriteAdminRouter.Handle("/admin/evacuateRoom/{roomID}", + httputil.MakeAuthAPI("admin_evacuate_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + if device.AccountType != userapi.AccountTypeAdmin { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("This API can only be used by admin users."), + } + } + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + roomID, ok := vars["roomID"] + if !ok { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.MissingArgument("Expecting room ID."), + } + } + res := &roomserverAPI.PerformAdminEvacuateRoomResponse{} + rsAPI.PerformAdminEvacuateRoom( + req.Context(), + &roomserverAPI.PerformAdminEvacuateRoomRequest{ + RoomID: roomID, + }, + res, + ) + if err := res.Error; err != nil { + return err.JSONResponse() + } + return util.JSONResponse{ + Code: 200, + JSON: map[string]interface{}{ + "affected": res.Affected, + }, + } + }), + ).Methods(http.MethodGet, http.MethodOptions) + // server notifications if cfg.Matrix.ServerNotices.Enabled { logrus.Info("Enabling server notices at /_synapse/admin/v1/send_server_notice") diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index dd1ab3697..7ec810c95 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -193,6 +193,7 @@ func main() { base.PublicWellKnownAPIMux, base.PublicMediaAPIMux, base.SynapseAdminMux, + base.DendriteAdminMux, ) wsUpgrader := websocket.Upgrader{ diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index b840eb2b8..54231f30c 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -150,6 +150,7 @@ func main() { base.PublicWellKnownAPIMux, base.PublicMediaAPIMux, base.SynapseAdminMux, + base.DendriteAdminMux, ) if err := mscs.Enable(base, &monolith); err != nil { logrus.WithError(err).Fatalf("Failed to enable MSCs") diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 1443ab5b1..5fd5c0b5d 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -153,6 +153,7 @@ func main() { base.PublicWellKnownAPIMux, base.PublicMediaAPIMux, base.SynapseAdminMux, + base.DendriteAdminMux, ) if len(base.Cfg.MSCs.MSCs) > 0 { diff --git a/cmd/dendrite-polylith-multi/personalities/clientapi.go b/cmd/dendrite-polylith-multi/personalities/clientapi.go index 1e509f88a..7ed2075aa 100644 --- a/cmd/dendrite-polylith-multi/personalities/clientapi.go +++ b/cmd/dendrite-polylith-multi/personalities/clientapi.go @@ -31,8 +31,10 @@ func ClientAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { keyAPI := base.KeyServerHTTPClient() clientapi.AddPublicRoutes( - base.ProcessContext, base.PublicClientAPIMux, base.SynapseAdminMux, &base.Cfg.ClientAPI, - federation, rsAPI, asQuery, transactions.New(), fsAPI, userAPI, userAPI, + base.ProcessContext, base.PublicClientAPIMux, + base.SynapseAdminMux, base.DendriteAdminMux, + &base.Cfg.ClientAPI, federation, rsAPI, asQuery, + transactions.New(), fsAPI, userAPI, userAPI, keyAPI, nil, &cfg.MSCs, ) diff --git a/cmd/dendritejs-pinecone/main.go b/cmd/dendritejs-pinecone/main.go index 211b3e131..0d4b2fbc5 100644 --- a/cmd/dendritejs-pinecone/main.go +++ b/cmd/dendritejs-pinecone/main.go @@ -220,6 +220,7 @@ func startup() { base.PublicWellKnownAPIMux, base.PublicMediaAPIMux, base.SynapseAdminMux, + base.DendriteAdminMux, ) httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() diff --git a/roomserver/api/api.go b/roomserver/api/api.go index fb77423f8..f0ca8a615 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -66,6 +66,12 @@ type RoomserverInternalAPI interface { res *PerformInboundPeekResponse, ) error + PerformAdminEvacuateRoom( + ctx context.Context, + req *PerformAdminEvacuateRoomRequest, + res *PerformAdminEvacuateRoomResponse, + ) + QueryPublishedRooms( ctx context.Context, req *QueryPublishedRoomsRequest, diff --git a/roomserver/api/api_trace.go b/roomserver/api/api_trace.go index ec7211ef8..61c06e886 100644 --- a/roomserver/api/api_trace.go +++ b/roomserver/api/api_trace.go @@ -104,6 +104,15 @@ func (t *RoomserverInternalAPITrace) PerformPublish( util.GetLogger(ctx).Infof("PerformPublish req=%+v res=%+v", js(req), js(res)) } +func (t *RoomserverInternalAPITrace) PerformAdminEvacuateRoom( + ctx context.Context, + req *PerformAdminEvacuateRoomRequest, + res *PerformAdminEvacuateRoomResponse, +) { + t.Impl.PerformAdminEvacuateRoom(ctx, req, res) + util.GetLogger(ctx).Infof("PerformAdminEvacuateRoom req=%+v res=%+v", js(req), js(res)) +} + func (t *RoomserverInternalAPITrace) PerformInboundPeek( ctx context.Context, req *PerformInboundPeekRequest, diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go index cda4b3ee4..30aa2cf1b 100644 --- a/roomserver/api/perform.go +++ b/roomserver/api/perform.go @@ -214,3 +214,12 @@ type PerformRoomUpgradeResponse struct { NewRoomID string Error *PerformError } + +type PerformAdminEvacuateRoomRequest struct { + RoomID string `json:"room_id"` +} + +type PerformAdminEvacuateRoomResponse struct { + Affected []string `json:"affected"` + Error *PerformError +} diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go index 59f485cf7..267cd4099 100644 --- a/roomserver/internal/api.go +++ b/roomserver/internal/api.go @@ -35,6 +35,7 @@ type RoomserverInternalAPI struct { *perform.Backfiller *perform.Forgetter *perform.Upgrader + *perform.Admin ProcessContext *process.ProcessContext DB storage.Database Cfg *config.RoomServer @@ -164,6 +165,12 @@ func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.FederationInternalA Cfg: r.Cfg, URSAPI: r, } + r.Admin = &perform.Admin{ + DB: r.DB, + Cfg: r.Cfg, + Inputer: r.Inputer, + Queryer: r.Queryer, + } if err := r.Inputer.Start(); err != nil { logrus.WithError(err).Panic("failed to start roomserver input API") diff --git a/roomserver/internal/perform/perform_admin.go b/roomserver/internal/perform/perform_admin.go new file mode 100644 index 000000000..2de6477cc --- /dev/null +++ b/roomserver/internal/perform/perform_admin.go @@ -0,0 +1,162 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package perform + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/matrix-org/dendrite/internal/eventutil" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/internal/input" + "github.com/matrix-org/dendrite/roomserver/internal/query" + "github.com/matrix-org/dendrite/roomserver/storage" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/gomatrixserverlib" +) + +type Admin struct { + DB storage.Database + Cfg *config.RoomServer + Queryer *query.Queryer + Inputer *input.Inputer +} + +// PerformEvacuateRoom will remove all local users from the given room. +func (r *Admin) PerformAdminEvacuateRoom( + ctx context.Context, + req *api.PerformAdminEvacuateRoomRequest, + res *api.PerformAdminEvacuateRoomResponse, +) { + roomInfo, err := r.DB.RoomInfo(ctx, req.RoomID) + if err != nil { + res.Error = &api.PerformError{ + Code: api.PerformErrorBadRequest, + Msg: fmt.Sprintf("r.DB.RoomInfo: %s", err), + } + return + } + if roomInfo == nil || roomInfo.IsStub { + res.Error = &api.PerformError{ + Code: api.PerformErrorNoRoom, + Msg: fmt.Sprintf("Room %s not found", req.RoomID), + } + return + } + + memberNIDs, err := r.DB.GetMembershipEventNIDsForRoom(ctx, roomInfo.RoomNID, true, true) + if err != nil { + res.Error = &api.PerformError{ + Code: api.PerformErrorBadRequest, + Msg: fmt.Sprintf("r.DB.GetMembershipEventNIDsForRoom: %s", err), + } + return + } + + memberEvents, err := r.DB.Events(ctx, memberNIDs) + if err != nil { + res.Error = &api.PerformError{ + Code: api.PerformErrorBadRequest, + Msg: fmt.Sprintf("r.DB.Events: %s", err), + } + return + } + + inputEvents := make([]api.InputRoomEvent, 0, len(memberEvents)) + res.Affected = make([]string, 0, len(memberEvents)) + latestReq := &api.QueryLatestEventsAndStateRequest{ + RoomID: req.RoomID, + } + latestRes := &api.QueryLatestEventsAndStateResponse{} + if err = r.Queryer.QueryLatestEventsAndState(ctx, latestReq, latestRes); err != nil { + res.Error = &api.PerformError{ + Code: api.PerformErrorBadRequest, + Msg: fmt.Sprintf("r.Queryer.QueryLatestEventsAndState: %s", err), + } + return + } + + prevEvents := latestRes.LatestEvents + for _, memberEvent := range memberEvents { + if memberEvent.StateKey() == nil { + continue + } + + var memberContent gomatrixserverlib.MemberContent + if err = json.Unmarshal(memberEvent.Content(), &memberContent); err != nil { + res.Error = &api.PerformError{ + Code: api.PerformErrorBadRequest, + Msg: fmt.Sprintf("json.Unmarshal: %s", err), + } + return + } + memberContent.Membership = gomatrixserverlib.Leave + + stateKey := *memberEvent.StateKey() + fledglingEvent := &gomatrixserverlib.EventBuilder{ + RoomID: req.RoomID, + Type: gomatrixserverlib.MRoomMember, + StateKey: &stateKey, + Sender: stateKey, + PrevEvents: prevEvents, + } + + if fledglingEvent.Content, err = json.Marshal(memberContent); err != nil { + res.Error = &api.PerformError{ + Code: api.PerformErrorBadRequest, + Msg: fmt.Sprintf("json.Marshal: %s", err), + } + return + } + + eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(fledglingEvent) + if err != nil { + res.Error = &api.PerformError{ + Code: api.PerformErrorBadRequest, + Msg: fmt.Sprintf("gomatrixserverlib.StateNeededForEventBuilder: %s", err), + } + return + } + + event, err := eventutil.BuildEvent(ctx, fledglingEvent, r.Cfg.Matrix, time.Now(), &eventsNeeded, latestRes) + if err != nil { + res.Error = &api.PerformError{ + Code: api.PerformErrorBadRequest, + Msg: fmt.Sprintf("eventutil.BuildEvent: %s", err), + } + return + } + + inputEvents = append(inputEvents, api.InputRoomEvent{ + Kind: api.KindNew, + Event: event, + Origin: r.Cfg.Matrix.ServerName, + SendAsServer: string(r.Cfg.Matrix.ServerName), + }) + res.Affected = append(res.Affected, stateKey) + prevEvents = []gomatrixserverlib.EventReference{ + event.EventReference(), + } + } + + inputReq := &api.InputRoomEventsRequest{ + InputRoomEvents: inputEvents, + Asynchronous: true, + } + inputRes := &api.InputRoomEventsResponse{} + r.Inputer.InputRoomEvents(ctx, inputReq, inputRes) +} diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go index d55805a91..3b29001e9 100644 --- a/roomserver/inthttp/client.go +++ b/roomserver/inthttp/client.go @@ -29,16 +29,17 @@ const ( RoomserverInputRoomEventsPath = "/roomserver/inputRoomEvents" // Perform operations - RoomserverPerformInvitePath = "/roomserver/performInvite" - RoomserverPerformPeekPath = "/roomserver/performPeek" - RoomserverPerformUnpeekPath = "/roomserver/performUnpeek" - RoomserverPerformRoomUpgradePath = "/roomserver/performRoomUpgrade" - RoomserverPerformJoinPath = "/roomserver/performJoin" - RoomserverPerformLeavePath = "/roomserver/performLeave" - RoomserverPerformBackfillPath = "/roomserver/performBackfill" - RoomserverPerformPublishPath = "/roomserver/performPublish" - RoomserverPerformInboundPeekPath = "/roomserver/performInboundPeek" - RoomserverPerformForgetPath = "/roomserver/performForget" + RoomserverPerformInvitePath = "/roomserver/performInvite" + RoomserverPerformPeekPath = "/roomserver/performPeek" + RoomserverPerformUnpeekPath = "/roomserver/performUnpeek" + RoomserverPerformRoomUpgradePath = "/roomserver/performRoomUpgrade" + RoomserverPerformJoinPath = "/roomserver/performJoin" + RoomserverPerformLeavePath = "/roomserver/performLeave" + RoomserverPerformBackfillPath = "/roomserver/performBackfill" + RoomserverPerformPublishPath = "/roomserver/performPublish" + RoomserverPerformInboundPeekPath = "/roomserver/performInboundPeek" + RoomserverPerformForgetPath = "/roomserver/performForget" + RoomserverPerformAdminEvacuateRoomPath = "/roomserver/performAdminEvacuateRoom" // Query operations RoomserverQueryLatestEventsAndStatePath = "/roomserver/queryLatestEventsAndState" @@ -299,6 +300,23 @@ func (h *httpRoomserverInternalAPI) PerformPublish( } } +func (h *httpRoomserverInternalAPI) PerformAdminEvacuateRoom( + ctx context.Context, + req *api.PerformAdminEvacuateRoomRequest, + res *api.PerformAdminEvacuateRoomResponse, +) { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformAdminEvacuateRoom") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverPerformAdminEvacuateRoomPath + err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res) + if err != nil { + res.Error = &api.PerformError{ + Msg: fmt.Sprintf("failed to communicate with roomserver: %s", err), + } + } +} + // QueryLatestEventsAndState implements RoomserverQueryAPI func (h *httpRoomserverInternalAPI) QueryLatestEventsAndState( ctx context.Context, diff --git a/roomserver/inthttp/server.go b/roomserver/inthttp/server.go index 0b27b5a8d..c5159a63c 100644 --- a/roomserver/inthttp/server.go +++ b/roomserver/inthttp/server.go @@ -118,6 +118,17 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + internalAPIMux.Handle(RoomserverPerformAdminEvacuateRoomPath, + httputil.MakeInternalAPI("performAdminEvacuateRoom", func(req *http.Request) util.JSONResponse { + var request api.PerformAdminEvacuateRoomRequest + var response api.PerformAdminEvacuateRoomResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + r.PerformAdminEvacuateRoom(req.Context(), &request, &response) + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) internalAPIMux.Handle( RoomserverQueryPublishedRoomsPath, httputil.MakeInternalAPI("queryPublishedRooms", func(req *http.Request) util.JSONResponse { diff --git a/setup/monolith.go b/setup/monolith.go index 32f1a6494..c86ec7b69 100644 --- a/setup/monolith.go +++ b/setup/monolith.go @@ -54,13 +54,13 @@ type Monolith struct { } // AddAllPublicRoutes attaches all public paths to the given router -func (m *Monolith) AddAllPublicRoutes(process *process.ProcessContext, csMux, ssMux, keyMux, wkMux, mediaMux, synapseMux *mux.Router) { +func (m *Monolith) AddAllPublicRoutes(process *process.ProcessContext, csMux, ssMux, keyMux, wkMux, mediaMux, synapseMux, dendriteMux *mux.Router) { userDirectoryProvider := m.ExtUserDirectoryProvider if userDirectoryProvider == nil { userDirectoryProvider = m.UserAPI } clientapi.AddPublicRoutes( - process, csMux, synapseMux, &m.Config.ClientAPI, + process, csMux, synapseMux, dendriteMux, &m.Config.ClientAPI, m.FedClient, m.RoomserverAPI, m.AppserviceAPI, transactions.New(), m.FederationAPI, m.UserAPI, userDirectoryProvider, m.KeyAPI, From 2a5b8e0306a283aa8ca64822d59d71479ffba59a Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Thu, 28 Apr 2022 18:53:28 +0200 Subject: [PATCH 41/50] Only load members of newly joined rooms (#2389) * Only load members of newly joined rooms * Comment that the query is prepared at runtime Co-authored-by: Neil Alexander --- syncapi/notifier/notifier.go | 14 ++++++ syncapi/storage/interface.go | 3 ++ .../postgres/current_room_state_table.go | 34 ++++++++++++- syncapi/storage/shared/syncserver.go | 4 ++ .../sqlite3/current_room_state_table.go | 49 +++++++++++++++++-- syncapi/storage/tables/interface.go | 2 + syncapi/streams/stream_presence.go | 5 +- 7 files changed, 102 insertions(+), 9 deletions(-) diff --git a/syncapi/notifier/notifier.go b/syncapi/notifier/notifier.go index 82834239b..87f0d86d7 100644 --- a/syncapi/notifier/notifier.go +++ b/syncapi/notifier/notifier.go @@ -333,6 +333,20 @@ func (n *Notifier) Load(ctx context.Context, db storage.Database) error { return nil } +// LoadRooms loads the membership states required to notify users correctly. +func (n *Notifier) LoadRooms(ctx context.Context, db storage.Database, roomIDs []string) error { + n.lock.Lock() + defer n.lock.Unlock() + + roomToUsers, err := db.AllJoinedUsersInRoom(ctx, roomIDs) + if err != nil { + return err + } + n.setUsersJoinedToRooms(roomToUsers) + + return nil +} + // CurrentPosition returns the current sync position func (n *Notifier) CurrentPosition() types.StreamingToken { n.lock.RLock() diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 486978598..5a036d889 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -52,6 +52,9 @@ type Database interface { // AllJoinedUsersInRooms returns a map of room ID to a list of all joined user IDs. AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) + // AllJoinedUsersInRoom returns a map of room ID to a list of all joined user IDs for a given room. + AllJoinedUsersInRoom(ctx context.Context, roomIDs []string) (map[string][]string, error) + // AllPeekingDevicesInRooms returns a map of room ID to a list of all peeking devices. AllPeekingDevicesInRooms(ctx context.Context) (map[string][]types.PeekingDevice, error) // Events lookups a list of event by their event ID. diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index fe68788d1..8ee387b39 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -93,6 +93,9 @@ const selectCurrentStateSQL = "" + const selectJoinedUsersSQL = "" + "SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join'" +const selectJoinedUsersInRoomSQL = "" + + "SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join' AND room_id = ANY($1)" + const selectStateEventSQL = "" + "SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3" @@ -112,6 +115,7 @@ type currentRoomStateStatements struct { selectRoomIDsWithAnyMembershipStmt *sql.Stmt selectCurrentStateStmt *sql.Stmt selectJoinedUsersStmt *sql.Stmt + selectJoinedUsersInRoomStmt *sql.Stmt selectEventsWithEventIDsStmt *sql.Stmt selectStateEventStmt *sql.Stmt } @@ -143,6 +147,9 @@ func NewPostgresCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, erro if s.selectJoinedUsersStmt, err = db.Prepare(selectJoinedUsersSQL); err != nil { return nil, err } + if s.selectJoinedUsersInRoomStmt, err = db.Prepare(selectJoinedUsersInRoomSQL); err != nil { + return nil, err + } if s.selectEventsWithEventIDsStmt, err = db.Prepare(selectEventsWithEventIDsSQL); err != nil { return nil, err } @@ -163,9 +170,32 @@ func (s *currentRoomStateStatements) SelectJoinedUsers( defer internal.CloseAndLogIfError(ctx, rows, "selectJoinedUsers: rows.close() failed") result := make(map[string][]string) + var roomID string + var userID string + for rows.Next() { + if err := rows.Scan(&roomID, &userID); err != nil { + return nil, err + } + users := result[roomID] + users = append(users, userID) + result[roomID] = users + } + return result, rows.Err() +} + +// SelectJoinedUsersInRoom returns a map of room ID to a list of joined user IDs for a given room. +func (s *currentRoomStateStatements) SelectJoinedUsersInRoom( + ctx context.Context, roomIDs []string, +) (map[string][]string, error) { + rows, err := s.selectJoinedUsersInRoomStmt.QueryContext(ctx, pq.StringArray(roomIDs)) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "selectJoinedUsers: rows.close() failed") + + result := make(map[string][]string) + var userID, roomID string for rows.Next() { - var roomID string - var userID string if err := rows.Scan(&roomID, &userID); err != nil { return nil, err } diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index b7d2d3a29..ec5edd355 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -168,6 +168,10 @@ func (d *Database) AllJoinedUsersInRooms(ctx context.Context) (map[string][]stri return d.CurrentRoomState.SelectJoinedUsers(ctx) } +func (d *Database) AllJoinedUsersInRoom(ctx context.Context, roomIDs []string) (map[string][]string, error) { + return d.CurrentRoomState.SelectJoinedUsersInRoom(ctx, roomIDs) +} + func (d *Database) AllPeekingDevicesInRooms(ctx context.Context) (map[string][]types.PeekingDevice, error) { return d.Peeks.SelectPeekingDevices(ctx) } diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index ccda005c1..f0a1c7bb7 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -77,6 +77,9 @@ const selectCurrentStateSQL = "" + const selectJoinedUsersSQL = "" + "SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join'" +const selectJoinedUsersInRoomSQL = "" + + "SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join' AND room_id IN ($1)" + const selectStateEventSQL = "" + "SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3" @@ -97,7 +100,8 @@ type currentRoomStateStatements struct { selectRoomIDsWithMembershipStmt *sql.Stmt selectRoomIDsWithAnyMembershipStmt *sql.Stmt selectJoinedUsersStmt *sql.Stmt - selectStateEventStmt *sql.Stmt + //selectJoinedUsersInRoomStmt *sql.Stmt - prepared at runtime due to variadic + selectStateEventStmt *sql.Stmt } func NewSqliteCurrentRoomStateTable(db *sql.DB, streamID *StreamIDStatements) (tables.CurrentRoomState, error) { @@ -127,13 +131,16 @@ func NewSqliteCurrentRoomStateTable(db *sql.DB, streamID *StreamIDStatements) (t if s.selectJoinedUsersStmt, err = db.Prepare(selectJoinedUsersSQL); err != nil { return nil, err } + //if s.selectJoinedUsersInRoomStmt, err = db.Prepare(selectJoinedUsersInRoomSQL); err != nil { + // return nil, err + //} if s.selectStateEventStmt, err = db.Prepare(selectStateEventSQL); err != nil { return nil, err } return s, nil } -// JoinedMemberLists returns a map of room ID to a list of joined user IDs. +// SelectJoinedUsers returns a map of room ID to a list of joined user IDs. func (s *currentRoomStateStatements) SelectJoinedUsers( ctx context.Context, ) (map[string][]string, error) { @@ -144,9 +151,9 @@ func (s *currentRoomStateStatements) SelectJoinedUsers( defer internal.CloseAndLogIfError(ctx, rows, "selectJoinedUsers: rows.close() failed") result := make(map[string][]string) + var roomID string + var userID string for rows.Next() { - var roomID string - var userID string if err := rows.Scan(&roomID, &userID); err != nil { return nil, err } @@ -157,6 +164,40 @@ func (s *currentRoomStateStatements) SelectJoinedUsers( return result, nil } +// SelectJoinedUsersInRoom returns a map of room ID to a list of joined user IDs for a given room. +func (s *currentRoomStateStatements) SelectJoinedUsersInRoom( + ctx context.Context, roomIDs []string, +) (map[string][]string, error) { + query := strings.Replace(selectJoinedUsersInRoomSQL, "($1)", sqlutil.QueryVariadic(len(roomIDs)), 1) + params := make([]interface{}, 0, len(roomIDs)) + for _, roomID := range roomIDs { + params = append(params, roomID) + } + stmt, err := s.db.Prepare(query) + if err != nil { + return nil, fmt.Errorf("SelectJoinedUsersInRoom s.db.Prepare: %w", err) + } + defer internal.CloseAndLogIfError(ctx, stmt, "SelectJoinedUsersInRoom: stmt.close() failed") + + rows, err := stmt.QueryContext(ctx, params...) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "SelectJoinedUsersInRoom: rows.close() failed") + + result := make(map[string][]string) + var userID, roomID string + for rows.Next() { + if err := rows.Scan(&roomID, &userID); err != nil { + return nil, err + } + users := result[roomID] + users = append(users, userID) + result[roomID] = users + } + return result, rows.Err() +} + // SelectRoomIDsWithMembership returns the list of room IDs which have the given user in the given membership state. func (s *currentRoomStateStatements) SelectRoomIDsWithMembership( ctx context.Context, diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index 6fdd9483e..ccdebfdbd 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -102,6 +102,8 @@ type CurrentRoomState interface { SelectRoomIDsWithAnyMembership(ctx context.Context, txn *sql.Tx, userID string) (map[string]string, error) // SelectJoinedUsers returns a map of room ID to a list of joined user IDs. SelectJoinedUsers(ctx context.Context) (map[string][]string, error) + // SelectJoinedUsersInRoom returns a map of room ID to a list of joined user IDs for a given room. + SelectJoinedUsersInRoom(ctx context.Context, roomIDs []string) (map[string][]string, error) } // BackwardsExtremities keeps track of backwards extremities for a room. diff --git a/syncapi/streams/stream_presence.go b/syncapi/streams/stream_presence.go index 675a7a178..a84d19878 100644 --- a/syncapi/streams/stream_presence.go +++ b/syncapi/streams/stream_presence.go @@ -67,9 +67,8 @@ func (p *PresenceStreamProvider) IncrementalSync( // add newly joined rooms user presences newlyJoined := joinedRooms(req.Response, req.Device.UserID) if len(newlyJoined) > 0 { - // TODO: This refreshes all lists and is quite expensive - // The notifier should update the lists itself - if err = p.notifier.Load(ctx, p.DB); err != nil { + // TODO: Check if this is working better than before. + if err = p.notifier.LoadRooms(ctx, p.DB, newlyJoined); err != nil { req.Log.WithError(err).Error("unable to refresh notifier lists") return from } From 26a1512808282c954a141b3376c47b05ef1e6ab4 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Fri, 29 Apr 2022 09:31:11 +0200 Subject: [PATCH 42/50] Add restrictions for open registration (#2402) * Add restrications for open registration * Make enable open registration a parameter * Enable registration for CI * Update error message * Shuffle things around a bit * Add a warning at every startup just to be extra annoying * Ignore shared secret when warning about open registration, since it's not strictly required when it is set if registration is otherwise enabled * Make CI happy? * Add missing parameter; try new parameter in upgrade-test Co-authored-by: Neil Alexander --- build/docker/config/dendrite.yaml | 2 +- build/gobind-pinecone/monolith.go | 2 ++ build/gobind-yggdrasil/monolith.go | 2 ++ build/scripts/Complement.Dockerfile | 2 +- build/scripts/ComplementLocal.Dockerfile | 2 +- build/scripts/ComplementPostgres.Dockerfile | 2 +- cmd/dendrite-demo-pinecone/main.go | 2 ++ cmd/dendrite-demo-yggdrasil/main.go | 2 ++ cmd/dendrite-upgrade-tests/main.go | 3 ++- cmd/dendritejs-pinecone/main.go | 2 ++ cmd/generate-config/main.go | 2 ++ dendrite-config.yaml | 2 +- setup/base/base.go | 4 ++++ setup/config/config_clientapi.go | 23 ++++++++++++++++++++- setup/flags.go | 9 ++++++-- 15 files changed, 52 insertions(+), 9 deletions(-) diff --git a/build/docker/config/dendrite.yaml b/build/docker/config/dendrite.yaml index e3a0316dc..94dcf4558 100644 --- a/build/docker/config/dendrite.yaml +++ b/build/docker/config/dendrite.yaml @@ -140,7 +140,7 @@ client_api: # Prevents new users from being able to register on this homeserver, except when # using the registration shared secret below. - registration_disabled: false + registration_disabled: true # If set, allows registration by anyone who knows the shared secret, regardless of # whether registration is otherwise disabled. diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 6b2533491..d047f3fff 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -259,6 +259,8 @@ func (m *DendriteMonolith) Start() { cfg.MediaAPI.BasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory)) cfg.MediaAPI.AbsBasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory)) cfg.MSCs.MSCs = []string{"msc2836", "msc2946"} + cfg.ClientAPI.RegistrationDisabled = false + cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true if err := cfg.Derive(); err != nil { panic(err) } diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go index b9c6c1b78..4e95e3972 100644 --- a/build/gobind-yggdrasil/monolith.go +++ b/build/gobind-yggdrasil/monolith.go @@ -97,6 +97,8 @@ func (m *DendriteMonolith) Start() { cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-appservice.db", m.StorageDirectory)) cfg.MediaAPI.BasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory)) cfg.MediaAPI.AbsBasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory)) + cfg.ClientAPI.RegistrationDisabled = false + cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true if err = cfg.Derive(); err != nil { panic(err) } diff --git a/build/scripts/Complement.Dockerfile b/build/scripts/Complement.Dockerfile index 6b2942d97..63e3890ee 100644 --- a/build/scripts/Complement.Dockerfile +++ b/build/scripts/Complement.Dockerfile @@ -29,4 +29,4 @@ EXPOSE 8008 8448 CMD ./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \ ./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \ cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \ - ./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0} + ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0} diff --git a/build/scripts/ComplementLocal.Dockerfile b/build/scripts/ComplementLocal.Dockerfile index 60b4d983a..a9feb4cd1 100644 --- a/build/scripts/ComplementLocal.Dockerfile +++ b/build/scripts/ComplementLocal.Dockerfile @@ -32,7 +32,7 @@ RUN echo '\ ./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key \n\ ./generate-config -server $SERVER_NAME --ci > dendrite.yaml \n\ cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates \n\ -./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\ +./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\ ' > run.sh && chmod +x run.sh diff --git a/build/scripts/ComplementPostgres.Dockerfile b/build/scripts/ComplementPostgres.Dockerfile index b98f4671c..4e26faa58 100644 --- a/build/scripts/ComplementPostgres.Dockerfile +++ b/build/scripts/ComplementPostgres.Dockerfile @@ -51,4 +51,4 @@ CMD /build/run_postgres.sh && ./generate-keys --server $SERVER_NAME --tls-cert s sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml && \ sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml && \ cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \ - ./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0} \ No newline at end of file + ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0} \ No newline at end of file diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index 7ec810c95..785e7b460 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -140,6 +140,8 @@ func main() { cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName)) cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) cfg.MSCs.MSCs = []string{"msc2836", "msc2946"} + cfg.ClientAPI.RegistrationDisabled = false + cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true if err := cfg.Derive(); err != nil { panic(err) } diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 54231f30c..f9234319a 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -89,6 +89,8 @@ func main() { cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) cfg.MSCs.MSCs = []string{"msc2836"} cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName)) + cfg.ClientAPI.RegistrationDisabled = false + cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true if err = cfg.Derive(); err != nil { panic(err) } diff --git a/cmd/dendrite-upgrade-tests/main.go b/cmd/dendrite-upgrade-tests/main.go index 3241234ac..b7e7da07d 100644 --- a/cmd/dendrite-upgrade-tests/main.go +++ b/cmd/dendrite-upgrade-tests/main.go @@ -83,7 +83,8 @@ do \n\ done \n\ \n\ sed -i "s/server_name: localhost/server_name: ${SERVER_NAME}/g" dendrite.yaml \n\ -./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\ +PARAMS="--tls-cert server.crt --tls-key server.key --config dendrite.yaml" \n\ +./dendrite-monolith-server --really-enable-open-registration ${PARAMS} || ./dendrite-monolith-server ${PARAMS} \n\ ' > run_dendrite.sh && chmod +x run_dendrite.sh ENV SERVER_NAME=localhost diff --git a/cmd/dendritejs-pinecone/main.go b/cmd/dendritejs-pinecone/main.go index 0d4b2fbc5..5ecf1f2fb 100644 --- a/cmd/dendritejs-pinecone/main.go +++ b/cmd/dendritejs-pinecone/main.go @@ -171,6 +171,8 @@ func startup() { cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID) cfg.Global.PrivateKey = sk cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk)) + cfg.ClientAPI.RegistrationDisabled = false + cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true if err := cfg.Derive(); err != nil { logrus.Fatalf("Failed to derive values from config: %s", err) diff --git a/cmd/generate-config/main.go b/cmd/generate-config/main.go index 24085afaa..1c585d916 100644 --- a/cmd/generate-config/main.go +++ b/cmd/generate-config/main.go @@ -90,6 +90,8 @@ func main() { cfg.Logging[0].Type = "std" cfg.UserAPI.BCryptCost = bcrypt.MinCost cfg.Global.JetStream.InMemory = true + cfg.ClientAPI.RegistrationDisabled = false + cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true cfg.ClientAPI.RegistrationSharedSecret = "complement" cfg.Global.Presence = config.PresenceOptions{ EnableInbound: true, diff --git a/dendrite-config.yaml b/dendrite-config.yaml index 47f08c4fd..1c11ef96d 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -159,7 +159,7 @@ client_api: # Prevents new users from being able to register on this homeserver, except when # using the registration shared secret below. - registration_disabled: false + registration_disabled: true # Prevents new guest accounts from being created. Guest registration is also # disabled implicitly by setting 'registration_disabled' above. diff --git a/setup/base/base.go b/setup/base/base.go index 7091c6ba5..4b771aa36 100644 --- a/setup/base/base.go +++ b/setup/base/base.go @@ -126,6 +126,10 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base logrus.Infof("Dendrite version %s", internal.VersionString()) + if !cfg.ClientAPI.RegistrationDisabled && cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled { + logrus.Warn("Open registration is enabled") + } + closer, err := cfg.SetupTracing("Dendrite" + componentName) if err != nil { logrus.WithError(err).Panicf("failed to start opentracing") diff --git a/setup/config/config_clientapi.go b/setup/config/config_clientapi.go index 4590e752b..6104ed8b9 100644 --- a/setup/config/config_clientapi.go +++ b/setup/config/config_clientapi.go @@ -15,6 +15,12 @@ type ClientAPI struct { // If set disables new users from registering (except via shared // secrets) RegistrationDisabled bool `yaml:"registration_disabled"` + + // Enable registration without captcha verification or shared secret. + // This option is populated by the -really-enable-open-registration + // command line parameter as it is not recommended. + OpenRegistrationWithoutVerificationEnabled bool `yaml:"-"` + // If set, allows registration by anyone who also has the shared // secret, even if registration is otherwise disabled. RegistrationSharedSecret string `yaml:"registration_shared_secret"` @@ -55,7 +61,8 @@ func (c *ClientAPI) Defaults(generate bool) { c.RecaptchaEnabled = false c.RecaptchaBypassSecret = "" c.RecaptchaSiteVerifyAPI = "" - c.RegistrationDisabled = false + c.RegistrationDisabled = true + c.OpenRegistrationWithoutVerificationEnabled = false c.RateLimiting.Defaults() } @@ -72,6 +79,20 @@ func (c *ClientAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { } c.TURN.Verify(configErrs) c.RateLimiting.Verify(configErrs) + + // Ensure there is any spam counter measure when enabling registration + if !c.RegistrationDisabled && !c.OpenRegistrationWithoutVerificationEnabled { + if !c.RecaptchaEnabled { + configErrs.Add( + "You have tried to enable open registration without any secondary verification methods " + + "(such as reCAPTCHA). By enabling open registration, you are SIGNIFICANTLY " + + "increasing the risk that your server will be used to send spam or abuse, and may result in " + + "your server being banned from some rooms. If you are ABSOLUTELY CERTAIN you want to do this, " + + "start Dendrite with the -really-enable-open-registration command line flag. Otherwise, you " + + "should set the registration_disabled option in your Dendrite config.", + ) + } + } } type TURN struct { diff --git a/setup/flags.go b/setup/flags.go index 281cf3392..a9dac61a1 100644 --- a/setup/flags.go +++ b/setup/flags.go @@ -25,8 +25,9 @@ import ( ) var ( - configPath = flag.String("config", "dendrite.yaml", "The path to the config file. For more information, see the config file in this repository.") - version = flag.Bool("version", false, "Shows the current version and exits immediately.") + configPath = flag.String("config", "dendrite.yaml", "The path to the config file. For more information, see the config file in this repository.") + version = flag.Bool("version", false, "Shows the current version and exits immediately.") + enableRegistrationWithoutVerification = flag.Bool("really-enable-open-registration", false, "This allows open registration without secondary verification (reCAPTCHA). This is NOT RECOMMENDED and will SIGNIFICANTLY increase the risk that your server will be used to send spam or conduct attacks, which may result in your server being banned from rooms.") ) // ParseFlags parses the commandline flags and uses them to create a config. @@ -48,5 +49,9 @@ func ParseFlags(monolith bool) *config.Dendrite { logrus.Fatalf("Invalid config file: %s", err) } + if *enableRegistrationWithoutVerification { + cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true + } + return cfg } From 1e083794ef0d2968eae9c874e621d199a2b62238 Mon Sep 17 00:00:00 2001 From: Brian Meek Date: Fri, 29 Apr 2022 00:55:35 -0700 Subject: [PATCH 43/50] Update golangci-lint, how it's installed, and added to the PATH (#2403) * switch to dendrite server * minor refactor to merge store code * Fix issue where m.room.name is being filtered by the dendrite server * refresh dendrite main * refresh dendrite main * missing merges from the last dendrite refresh * revert unwanted changes in dendrite.yaml * Update golangci-lint, how it's installed, and added to the PATH Co-authored-by: Tak Wai Wong Co-authored-by: tak-slashtalk <64229756+tak-slashtalk@users.noreply.github.com> --- build/scripts/find-lint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/scripts/find-lint.sh b/build/scripts/find-lint.sh index e3564ae38..820b8cc46 100755 --- a/build/scripts/find-lint.sh +++ b/build/scripts/find-lint.sh @@ -25,7 +25,7 @@ echo "Installing golangci-lint..." # Make a backup of go.{mod,sum} first cp go.mod go.mod.bak && cp go.sum go.sum.bak -go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.41.1 +go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.45.2 # Run linting echo "Looking for lint..." @@ -33,7 +33,7 @@ echo "Looking for lint..." # Capture exit code to ensure go.{mod,sum} is restored before exiting exit_code=0 -PATH="$PATH:${GOPATH:-~/go}/bin" golangci-lint run $args || exit_code=1 +PATH="$PATH:$(go env GOPATH)/bin" golangci-lint run $args || exit_code=1 # Restore go.{mod,sum} mv go.mod.bak go.mod && mv go.sum.bak go.sum From 0d4b8eadaa45ff65cddd4b882a156de33c4bed7e Mon Sep 17 00:00:00 2001 From: Till Faelligen Date: Fri, 29 Apr 2022 10:00:28 +0200 Subject: [PATCH 44/50] Add create-account to Getting started --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 75c827c33..2a8c36508 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,10 @@ $ cp dendrite-config.yaml dendrite.yaml # Build and run the server: $ ./bin/dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml + +# Create an user account (add -admin for an admin user). +# Specify the localpart only, e.g. 'alice' for '@alice:domain.com' +$ ./bin/create-account --config dendrite.yaml -username alice ``` Then point your favourite Matrix client at `http://localhost:8008` or `https://localhost:8448`. From 2a4517f8e6c7f68d8bd94f0b4640b142f2d3ac52 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 29 Apr 2022 09:10:08 +0100 Subject: [PATCH 45/50] Move admin functions into their own file in the client API --- clientapi/routing/admin.go | 49 ++++++++++++++++++++++++++++++++++++ clientapi/routing/routing.go | 35 +------------------------- 2 files changed, 50 insertions(+), 34 deletions(-) create mode 100644 clientapi/routing/admin.go diff --git a/clientapi/routing/admin.go b/clientapi/routing/admin.go new file mode 100644 index 000000000..31e431c78 --- /dev/null +++ b/clientapi/routing/admin.go @@ -0,0 +1,49 @@ +package routing + +import ( + "net/http" + + "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/internal/httputil" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/util" +) + +func AdminEvacuateRoom(req *http.Request, device *userapi.Device, rsAPI roomserverAPI.RoomserverInternalAPI) util.JSONResponse { + if device.AccountType != userapi.AccountTypeAdmin { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("This API can only be used by admin users."), + } + } + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + roomID, ok := vars["roomID"] + if !ok { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.MissingArgument("Expecting room ID."), + } + } + res := &roomserverAPI.PerformAdminEvacuateRoomResponse{} + rsAPI.PerformAdminEvacuateRoom( + req.Context(), + &roomserverAPI.PerformAdminEvacuateRoomRequest{ + RoomID: roomID, + }, + res, + ) + if err := res.Error; err != nil { + return err.JSONResponse() + } + return util.JSONResponse{ + Code: 200, + JSON: map[string]interface{}{ + "affected": res.Affected, + }, + } +} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index ec90b80db..ba1c76b81 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -122,40 +122,7 @@ func Setup( dendriteAdminRouter.Handle("/admin/evacuateRoom/{roomID}", httputil.MakeAuthAPI("admin_evacuate_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if device.AccountType != userapi.AccountTypeAdmin { - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: jsonerror.Forbidden("This API can only be used by admin users."), - } - } - vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) - } - roomID, ok := vars["roomID"] - if !ok { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.MissingArgument("Expecting room ID."), - } - } - res := &roomserverAPI.PerformAdminEvacuateRoomResponse{} - rsAPI.PerformAdminEvacuateRoom( - req.Context(), - &roomserverAPI.PerformAdminEvacuateRoomRequest{ - RoomID: roomID, - }, - res, - ) - if err := res.Error; err != nil { - return err.JSONResponse() - } - return util.JSONResponse{ - Code: 200, - JSON: map[string]interface{}{ - "affected": res.Affected, - }, - } + return AdminEvacuateRoom(req, device, rsAPI) }), ).Methods(http.MethodGet, http.MethodOptions) From d28d0ee66e22402bebd791a46de33c8bf3169e26 Mon Sep 17 00:00:00 2001 From: Brian Meek Date: Fri, 29 Apr 2022 09:32:58 +0100 Subject: [PATCH 46/50] Fix `TestThumbnailsStorage` failing when media results come back in non-deterministic order; silence expected error when tests are run multiple times against the same postgres database (cherry-picked from #2395) Signed-off-by: Brian Meek --- mediaapi/storage/storage_test.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/mediaapi/storage/storage_test.go b/mediaapi/storage/storage_test.go index 8d3403045..fa88cd8e7 100644 --- a/mediaapi/storage/storage_test.go +++ b/mediaapi/storage/storage_test.go @@ -123,11 +123,19 @@ func TestThumbnailsStorage(t *testing.T) { t.Fatalf("expected %d stored thumbnail metadata, got %d", len(thumbnails), len(gotMediadatas)) } for i := range gotMediadatas { - if !reflect.DeepEqual(thumbnails[i].MediaMetadata, gotMediadatas[i].MediaMetadata) { - t.Fatalf("expected metadata %+v, got %v", thumbnails[i].MediaMetadata, gotMediadatas[i].MediaMetadata) + // metadata may be returned in a different order than it was stored, perform a search + metaDataMatches := func() bool { + for _, t := range thumbnails { + if reflect.DeepEqual(t.MediaMetadata, gotMediadatas[i].MediaMetadata) && reflect.DeepEqual(t.ThumbnailSize, gotMediadatas[i].ThumbnailSize) { + return true + } + } + return false } - if !reflect.DeepEqual(thumbnails[i].ThumbnailSize, gotMediadatas[i].ThumbnailSize) { - t.Fatalf("expected metadata %+v, got %v", thumbnails[i].ThumbnailSize, gotMediadatas[i].ThumbnailSize) + + if !metaDataMatches() { + t.Fatalf("expected metadata %+v, got %+v", thumbnails[i].MediaMetadata, gotMediadatas[i].MediaMetadata) + } } }) From 31799a3b2a733192a656cfa77662599bd1568bdc Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 29 Apr 2022 16:02:55 +0100 Subject: [PATCH 47/50] Device list display name fixes (#2405) * Get device names from `unsigned` in `/user/devices` * Fix display name updates * Fix bug * Fix another bug --- federationapi/routing/devices.go | 8 ++- keyserver/internal/internal.go | 95 ++++++++++++++++---------------- 2 files changed, 54 insertions(+), 49 deletions(-) diff --git a/federationapi/routing/devices.go b/federationapi/routing/devices.go index 8890eac4b..57286fa90 100644 --- a/federationapi/routing/devices.go +++ b/federationapi/routing/devices.go @@ -20,6 +20,7 @@ import ( keyapi "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" + "github.com/tidwall/gjson" ) // GetUserDevices for the given user id @@ -69,9 +70,14 @@ func GetUserDevices( continue } + displayName := dev.DisplayName + if displayName == "" { + displayName = gjson.GetBytes(dev.DeviceKeys.KeyJSON, "unsigned.device_display_name").Str + } + device := gomatrixserverlib.RespUserDevice{ DeviceID: dev.DeviceID, - DisplayName: dev.DisplayName, + DisplayName: displayName, Keys: key, } diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index 1677cf8e3..e556f44b0 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -632,43 +632,55 @@ func (a *KeyInternalAPI) uploadLocalDeviceKeys(ctx context.Context, req *api.Per } var keysToStore []api.DeviceMessage - // assert that the user ID / device ID are not lying for each key - for _, key := range req.DeviceKeys { - var serverName gomatrixserverlib.ServerName - _, serverName, err = gomatrixserverlib.SplitID('@', key.UserID) - if err != nil { - continue // ignore invalid users - } - if serverName != a.ThisServer { - continue // ignore remote users - } - if len(key.KeyJSON) == 0 { - keysToStore = append(keysToStore, key.WithStreamID(0)) - continue // deleted keys don't need sanity checking - } - // check that the device in question actually exists in the user - // API before we try and store a key for it - if _, ok := existingDeviceMap[key.DeviceID]; !ok { - continue - } - gotUserID := gjson.GetBytes(key.KeyJSON, "user_id").Str - gotDeviceID := gjson.GetBytes(key.KeyJSON, "device_id").Str - if gotUserID == key.UserID && gotDeviceID == key.DeviceID { - keysToStore = append(keysToStore, key.WithStreamID(0)) - continue - } - - res.KeyError(key.UserID, key.DeviceID, &api.KeyError{ - Err: fmt.Sprintf( - "user_id or device_id mismatch: users: %s - %s, devices: %s - %s", - gotUserID, key.UserID, gotDeviceID, key.DeviceID, - ), - }) - } if req.OnlyDisplayNameUpdates { - // add the display name field from keysToStore into existingKeys - keysToStore = appendDisplayNames(existingKeys, keysToStore) + for _, existingKey := range existingKeys { + for _, newKey := range req.DeviceKeys { + switch { + case existingKey.UserID != newKey.UserID: + continue + case existingKey.DeviceID != newKey.DeviceID: + continue + case existingKey.DisplayName != newKey.DisplayName: + existingKey.DisplayName = newKey.DisplayName + } + } + keysToStore = append(keysToStore, existingKey) + } + } else { + // assert that the user ID / device ID are not lying for each key + for _, key := range req.DeviceKeys { + var serverName gomatrixserverlib.ServerName + _, serverName, err = gomatrixserverlib.SplitID('@', key.UserID) + if err != nil { + continue // ignore invalid users + } + if serverName != a.ThisServer { + continue // ignore remote users + } + if len(key.KeyJSON) == 0 { + keysToStore = append(keysToStore, key.WithStreamID(0)) + continue // deleted keys don't need sanity checking + } + // check that the device in question actually exists in the user + // API before we try and store a key for it + if _, ok := existingDeviceMap[key.DeviceID]; !ok { + continue + } + gotUserID := gjson.GetBytes(key.KeyJSON, "user_id").Str + gotDeviceID := gjson.GetBytes(key.KeyJSON, "device_id").Str + if gotUserID == key.UserID && gotDeviceID == key.DeviceID { + keysToStore = append(keysToStore, key.WithStreamID(0)) + continue + } + + res.KeyError(key.UserID, key.DeviceID, &api.KeyError{ + Err: fmt.Sprintf( + "user_id or device_id mismatch: users: %s - %s, devices: %s - %s", + gotUserID, key.UserID, gotDeviceID, key.DeviceID, + ), + }) + } } // store the device keys and emit changes @@ -764,16 +776,3 @@ func emitDeviceKeyChanges(producer KeyChangeProducer, existing, new []api.Device } return producer.ProduceKeyChanges(keysAdded) } - -func appendDisplayNames(existing, new []api.DeviceMessage) []api.DeviceMessage { - for i, existingDevice := range existing { - for _, newDevice := range new { - if existingDevice.DeviceID != newDevice.DeviceID { - continue - } - existingDevice.DisplayName = newDevice.DisplayName - existing[i] = existingDevice - } - } - return existing -} From 987d7adc5d4e34827759b206ac4cc17cce78e221 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Sat, 30 Apr 2022 00:07:50 +0200 Subject: [PATCH 48/50] Return "to", if we didn't return any presence events (#2407) Return correct stream position, if we didn't return any presence events --- syncapi/streams/stream_presence.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/syncapi/streams/stream_presence.go b/syncapi/streams/stream_presence.go index a84d19878..35ce53cb6 100644 --- a/syncapi/streams/stream_presence.go +++ b/syncapi/streams/stream_presence.go @@ -145,6 +145,10 @@ func (p *PresenceStreamProvider) IncrementalSync( p.cache.Store(cacheKey, presence) } + if len(req.Response.Presence.Events) == 0 { + return to + } + return lastPos } From bfa344e83191c49bdc9917ab7a8ec31b93c202e9 Mon Sep 17 00:00:00 2001 From: Brian Meek Date: Fri, 29 Apr 2022 15:23:11 -0700 Subject: [PATCH 49/50] Test_Devices, sqlite may return devices in different order, test should still pass (#2406) --- userapi/storage/storage_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go index 2eb57d0bc..ef25b8000 100644 --- a/userapi/storage/storage_test.go +++ b/userapi/storage/storage_test.go @@ -168,7 +168,7 @@ func Test_Devices(t *testing.T) { devices2, err := db.GetDevicesByID(ctx, deviceIDs) assert.NoError(t, err, "unable to get devices by id") - assert.Equal(t, devices, devices2) + assert.ElementsMatch(t, devices, devices2) // Update device newName := "new display name" From 979a551f1e2aeb9f3417df5e52a7279230b7a3ba Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Mon, 2 May 2022 10:47:16 +0200 Subject: [PATCH 50/50] Return `null` if MaxFileSizeBytes is 0 (#2409) * Return "null" if MaxFileSizeBytes is 0 * Add comment and nil check (better save than sorry) * Simplify config --- mediaapi/routing/download.go | 2 +- mediaapi/routing/routing.go | 8 ++++++-- mediaapi/routing/upload.go | 18 +++++++++--------- mediaapi/routing/upload_test.go | 5 ++--- setup/config/config_mediaapi.go | 6 +++--- 5 files changed, 21 insertions(+), 18 deletions(-) diff --git a/mediaapi/routing/download.go b/mediaapi/routing/download.go index 5f22a9461..10b25a5cd 100644 --- a/mediaapi/routing/download.go +++ b/mediaapi/routing/download.go @@ -551,7 +551,7 @@ func (r *downloadRequest) getRemoteFile( // If we do not have a record, we need to fetch the remote file first and then respond from the local file err := r.fetchRemoteFileAndStoreMetadata( ctx, client, - cfg.AbsBasePath, *cfg.MaxFileSizeBytes, db, + cfg.AbsBasePath, cfg.MaxFileSizeBytes, db, cfg.ThumbnailSizes, activeThumbnailGeneration, cfg.MaxThumbnailGenerators, ) diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index 0e1583991..97dfd3341 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -35,7 +35,7 @@ import ( // configResponse is the response to GET /_matrix/media/r0/config // https://matrix.org/docs/spec/client_server/latest#get-matrix-media-r0-config type configResponse struct { - UploadSize config.FileSizeBytes `json:"m.upload.size"` + UploadSize *config.FileSizeBytes `json:"m.upload.size"` } // Setup registers the media API HTTP handlers @@ -73,9 +73,13 @@ func Setup( if r := rateLimits.Limit(req); r != nil { return *r } + respondSize := &cfg.MaxFileSizeBytes + if cfg.MaxFileSizeBytes == 0 { + respondSize = nil + } return util.JSONResponse{ Code: http.StatusOK, - JSON: configResponse{UploadSize: *cfg.MaxFileSizeBytes}, + JSON: configResponse{UploadSize: respondSize}, } }) diff --git a/mediaapi/routing/upload.go b/mediaapi/routing/upload.go index 972c52af0..2175648ea 100644 --- a/mediaapi/routing/upload.go +++ b/mediaapi/routing/upload.go @@ -90,7 +90,7 @@ func parseAndValidateRequest(req *http.Request, cfg *config.MediaAPI, dev *usera Logger: util.GetLogger(req.Context()).WithField("Origin", cfg.Matrix.ServerName), } - if resErr := r.Validate(*cfg.MaxFileSizeBytes); resErr != nil { + if resErr := r.Validate(cfg.MaxFileSizeBytes); resErr != nil { return nil, resErr } @@ -148,20 +148,20 @@ func (r *uploadRequest) doUpload( // r.storeFileAndMetadata(ctx, tmpDir, ...) // before you return from doUpload else we will leak a temp file. We could make this nicer with a `WithTransaction` style of // nested function to guarantee either storage or cleanup. - if *cfg.MaxFileSizeBytes > 0 { - if *cfg.MaxFileSizeBytes+1 <= 0 { + if cfg.MaxFileSizeBytes > 0 { + if cfg.MaxFileSizeBytes+1 <= 0 { r.Logger.WithFields(log.Fields{ - "MaxFileSizeBytes": *cfg.MaxFileSizeBytes, + "MaxFileSizeBytes": cfg.MaxFileSizeBytes, }).Warnf("Configured MaxFileSizeBytes overflows int64, defaulting to %d bytes", config.DefaultMaxFileSizeBytes) - cfg.MaxFileSizeBytes = &config.DefaultMaxFileSizeBytes + cfg.MaxFileSizeBytes = config.DefaultMaxFileSizeBytes } - reqReader = io.LimitReader(reqReader, int64(*cfg.MaxFileSizeBytes)+1) + reqReader = io.LimitReader(reqReader, int64(cfg.MaxFileSizeBytes)+1) } hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(ctx, reqReader, cfg.AbsBasePath) if err != nil { r.Logger.WithError(err).WithFields(log.Fields{ - "MaxFileSizeBytes": *cfg.MaxFileSizeBytes, + "MaxFileSizeBytes": cfg.MaxFileSizeBytes, }).Warn("Error while transferring file") return &util.JSONResponse{ Code: http.StatusBadRequest, @@ -170,9 +170,9 @@ func (r *uploadRequest) doUpload( } // Check if temp file size exceeds max file size configuration - if *cfg.MaxFileSizeBytes > 0 && bytesWritten > types.FileSizeBytes(*cfg.MaxFileSizeBytes) { + if cfg.MaxFileSizeBytes > 0 && bytesWritten > types.FileSizeBytes(cfg.MaxFileSizeBytes) { fileutils.RemoveDir(tmpDir, r.Logger) // delete temp file - return requestEntityTooLargeJSONResponse(*cfg.MaxFileSizeBytes) + return requestEntityTooLargeJSONResponse(cfg.MaxFileSizeBytes) } // Look up the media by the file hash. If we already have the file but under a diff --git a/mediaapi/routing/upload_test.go b/mediaapi/routing/upload_test.go index b2c2f5a44..e04c010f8 100644 --- a/mediaapi/routing/upload_test.go +++ b/mediaapi/routing/upload_test.go @@ -36,12 +36,11 @@ func Test_uploadRequest_doUpload(t *testing.T) { } maxSize := config.FileSizeBytes(8) - unlimitedSize := config.FileSizeBytes(0) logger := log.New().WithField("mediaapi", "test") testdataPath := filepath.Join(wd, "./testdata") cfg := &config.MediaAPI{ - MaxFileSizeBytes: &maxSize, + MaxFileSizeBytes: maxSize, BasePath: config.Path(testdataPath), AbsBasePath: config.Path(testdataPath), DynamicThumbnails: false, @@ -124,7 +123,7 @@ func Test_uploadRequest_doUpload(t *testing.T) { ctx: context.Background(), reqReader: strings.NewReader("test test test"), cfg: &config.MediaAPI{ - MaxFileSizeBytes: &unlimitedSize, + MaxFileSizeBytes: config.FileSizeBytes(0), BasePath: config.Path(testdataPath), AbsBasePath: config.Path(testdataPath), DynamicThumbnails: false, diff --git a/setup/config/config_mediaapi.go b/setup/config/config_mediaapi.go index 9a7d84969..c85020d2a 100644 --- a/setup/config/config_mediaapi.go +++ b/setup/config/config_mediaapi.go @@ -23,7 +23,7 @@ type MediaAPI struct { // The maximum file size in bytes that is allowed to be stored on this server. // Note: if max_file_size_bytes is set to 0, the size is unlimited. // Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB) - MaxFileSizeBytes *FileSizeBytes `yaml:"max_file_size_bytes,omitempty"` + MaxFileSizeBytes FileSizeBytes `yaml:"max_file_size_bytes,omitempty"` // Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated DynamicThumbnails bool `yaml:"dynamic_thumbnails"` @@ -48,7 +48,7 @@ func (c *MediaAPI) Defaults(generate bool) { c.BasePath = "./media_store" } - c.MaxFileSizeBytes = &DefaultMaxFileSizeBytes + c.MaxFileSizeBytes = DefaultMaxFileSizeBytes c.MaxThumbnailGenerators = 10 } @@ -61,7 +61,7 @@ func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { checkNotEmpty(configErrs, "media_api.database.connection_string", string(c.Database.ConnectionString)) checkNotEmpty(configErrs, "media_api.base_path", string(c.BasePath)) - checkPositive(configErrs, "media_api.max_file_size_bytes", int64(*c.MaxFileSizeBytes)) + checkPositive(configErrs, "media_api.max_file_size_bytes", int64(c.MaxFileSizeBytes)) checkPositive(configErrs, "media_api.max_thumbnail_generators", int64(c.MaxThumbnailGenerators)) for i, size := range c.ThumbnailSizes {