diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index 99747f798..8221bff96 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -73,6 +73,26 @@ jobs: timeout-minutes: 5 name: Unit tests (Go ${{ matrix.go }}) runs-on: ubuntu-latest + # Service containers to run with `container-job` + services: + # Label used to access the service container + postgres: + # Docker Hub image + image: postgres:13-alpine + # Provide the password for postgres + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: dendrite + ports: + # Maps tcp port 5432 on service container to the host + - 5432:5432 + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 strategy: fail-fast: false matrix: @@ -91,7 +111,12 @@ jobs: key: ${{ runner.os }}-go${{ matrix.go }}-test-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-go${{ matrix.go }}-test- - - run: go test ./... + - run: go test -p 1 ./... + env: + POSTGRES_HOST: localhost + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: dendrite # build Dendrite for linux with different architectures and go versions build: @@ -233,7 +258,14 @@ jobs: - name: Summarise results.tap if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - + - name: Sytest List Maintenance + if: ${{ always() }} + run: /src/show-expected-fail-tests.sh /logs/results.tap /src/sytest-whitelist /src/sytest-blacklist + continue-on-error: true # not fatal + - name: Are We Synapse Yet? + if: ${{ always() }} + run: /src/are-we-synapse-yet.py /logs/results.tap -v + continue-on-error: true # not fatal - name: Upload Sytest logs uses: actions/upload-artifact@v2 if: ${{ always() }} diff --git a/CHANGES.md b/CHANGES.md index 428ecbf2a..831a8969d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,32 @@ # Changelog +## Dendrite 0.8.1 (2022-04-07) + +### Fixes + +* A bug which could result in the sync API deadlocking due to lock contention in the notifier has been fixed + +## Dendrite 0.8.0 (2022-04-07) + +### Features + +* Support for presence has been added + * Presence is not enabled by default + * The `global.presence.enable_inbound` and `global.presence.enable_outbound` configuration options allow configuring inbound and outbound presence separately +* Support for room upgrades via the `/room/{roomID}/upgrade` endpoint has been added (contributed by [DavidSpenler](https://github.com/DavidSpenler), [alexkursell](https://github.com/alexkursell)) +* Support for ignoring users has been added +* Joined and invite user counts are now sent in the `/sync` room summaries +* Queued federation and stale device list updates will now be staggered at startup over an up-to 2 minute warm-up period, rather than happening all at once +* Memory pressure created by the sync notifier has been reduced +* The EDU server component has now been removed, with the work being moved to more relevant components + +### Fixes + +* It is now possible to set the `power_level_content_override` when creating a room to include power levels over 100 +* `/send_join` and `/state` responses will now not unmarshal the JSON twice +* The stream event consumer for push notifications will no longer request membership events that are irrelevant +* Appservices will no longer incorrectly receive state events twice + ## Dendrite 0.7.0 (2022-03-25) ### Features diff --git a/README.md b/README.md index b4a9a614a..cbb35ad59 100644 --- a/README.md +++ b/README.md @@ -82,20 +82,17 @@ Then point your favourite Matrix client at `http://localhost:8008` or `https://l We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it -updates with CI. As of March 2022 we're at around 76% CS API coverage and 95% Federation coverage, though check +updates with CI. As of April 2022 we're at around 83% CS API coverage and 95% Federation coverage, though check CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse -servers such as matrix.org reasonably well. There's a long list of features that are not implemented, notably: - -- Search -- User Directory -- Presence +servers such as matrix.org reasonably well, although there are still some missing features (like Search). We are prioritising features that will benefit single-user homeservers first (e.g Receipts, E2E) rather than features that massive deployments may be interested in (User Directory, OpenID, Guests, Admin APIs, AS API). This means Dendrite supports amongst others: - Core room functionality (creating rooms, invites, auth rules) -- Federation in rooms v1-v7 +- Full support for room versions 1 to 7 +- Experimental support for room versions 8 to 9 - Backfilling locally and via federation - Accounts, Profiles and Devices - Published room lists @@ -108,6 +105,8 @@ This means Dendrite supports amongst others: - Receipts - Push - Guests +- User Directory +- Presence ## Contributing diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index 01790722a..31e05caa0 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -83,20 +83,29 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msg *nats.Msg) return true } - if output.Type != api.OutputTypeNewRoomEvent { + if output.Type != api.OutputTypeNewRoomEvent || output.NewRoomEvent == nil { return true } - events := []*gomatrixserverlib.HeaderedEvent{output.NewRoomEvent.Event} + newEventID := output.NewRoomEvent.Event.EventID() + events := make([]*gomatrixserverlib.HeaderedEvent, 0, len(output.NewRoomEvent.AddsStateEventIDs)) + events = append(events, output.NewRoomEvent.Event) if len(output.NewRoomEvent.AddsStateEventIDs) > 0 { eventsReq := &api.QueryEventsByIDRequest{ - EventIDs: output.NewRoomEvent.AddsStateEventIDs, + EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)), } eventsRes := &api.QueryEventsByIDResponse{} - if err := s.rsAPI.QueryEventsByID(s.ctx, eventsReq, eventsRes); err != nil { - return false + for _, eventID := range output.NewRoomEvent.AddsStateEventIDs { + if eventID != newEventID { + eventsReq.EventIDs = append(eventsReq.EventIDs, eventID) + } + } + if len(eventsReq.EventIDs) > 0 { + if err := s.rsAPI.QueryEventsByID(s.ctx, eventsReq, eventsRes); err != nil { + return false + } + events = append(events, eventsRes.Events...) } - events = append(events, eventsRes.Events...) } // Send event to any relevant application services diff --git a/are-we-synapse-yet.list b/are-we-synapse-yet.list index c2c726e57..4281171ab 100644 --- a/are-we-synapse-yet.list +++ b/are-we-synapse-yet.list @@ -106,10 +106,13 @@ rst Users cannot set notifications powerlevel higher than their own (2 subtests) rst Both GET and PUT work rct POST /rooms/:room_id/receipt can create receipts red POST /rooms/:room_id/read_markers can create read marker +med POST /media/v3/upload can create an upload med POST /media/r0/upload can create an upload +med GET /media/v3/download can fetch the value again med GET /media/r0/download can fetch the value again cap GET /capabilities is present and well formed for registered user cap GET /r0/capabilities is not public +cap GET /v3/capabilities is not public reg Register with a recaptcha reg registration is idempotent, without username specified reg registration is idempotent, with username specified @@ -174,7 +177,9 @@ eph Ephemeral messages received from clients are correctly expired ali Room aliases can contain Unicode f,ali Remote room alias queries can handle Unicode ali Canonical alias can be set +ali Canonical alias can be set (3 subtests) ali Canonical alias can include alt_aliases +ali Canonical alias can include alt_aliases (4 subtests) ali Regular users can add and delete aliases in the default room configuration ali Regular users can add and delete aliases when m.room.aliases is restricted ali Deleting a non-existent alias should return a 404 @@ -478,6 +483,30 @@ rmv Inbound federation rejects invites which include invalid JSON for room versi rmv Outbound federation rejects invite response which include invalid JSON for room version 6 rmv Inbound federation rejects invite rejections which include invalid JSON for room version 6 rmv Server rejects invalid JSON in a version 6 room +rmv User can create and send/receive messages in a room with version 7 (2 subtests) +rmv local user can join room with version 7 +rmv User can invite local user to room with version 7 +rmv remote user can join room with version 7 +rmv User can invite remote user to room with version 7 +rmv Remote user can backfill in a room with version 7 +rmv Can reject invites over federation for rooms with version 7 +rmv Can receive redactions from regular users over federation in room version 7 +rmv User can create and send/receive messages in a room with version 8 (2 subtests) +rmv local user can join room with version 8 +rmv User can invite local user to room with version 8 +rmv remote user can join room with version 8 +rmv User can invite remote user to room with version 8 +rmv Remote user can backfill in a room with version 8 +rmv Can reject invites over federation for rooms with version 8 +rmv Can receive redactions from regular users over federation in room version 8 +rmv User can create and send/receive messages in a room with version 9 (2 subtests) +rmv local user can join room with version 9 +rmv User can invite local user to room with version 9 +rmv remote user can join room with version 9 +rmv User can invite remote user to room with version 9 +rmv Remote user can backfill in a room with version 9 +rmv Can reject invites over federation for rooms with version 9 +rmv Can receive redactions from regular users over federation in room version 9 pre Presence changes are reported to local room members f,pre Presence changes are also reported to remote room members pre Presence changes to UNAVAILABLE are reported to local room members @@ -772,12 +801,15 @@ app AS can make room aliases app Regular users cannot create room aliases within the AS namespace app AS-ghosted users can use rooms via AS app AS-ghosted users can use rooms themselves +app AS-ghosted users can use rooms via AS (2 subtests) +app AS-ghosted users can use rooms themselves (3 subtests) app Ghost user must register before joining room app AS can set avatar for ghosted users app AS can set displayname for ghosted users app AS can't set displayname for random users app Inviting an AS-hosted user asks the AS server app Accesing an AS-hosted room alias asks the AS server +app Accesing an AS-hosted room alias asks the AS server (2 subtests) app Events in rooms with AS-hosted room aliases are sent to AS server app AS user (not ghost) can join room without registering app AS user (not ghost) can join room without registering, with user_id query param @@ -789,6 +821,8 @@ app AS can publish rooms in their own list app AS and main public room lists are separate app AS can deactivate a user psh Test that a message is pushed +psh Test that a message is pushed (6 subtests) +psh Test that rejected pushers are removed. (4 subtests) psh Invites are pushed psh Rooms with names are correctly named in pushed psh Rooms with canonical alias are correctly named in pushed @@ -857,9 +891,12 @@ pre Left room members do not cause problems for presence crm Rooms can be created with an initial invite list (SYN-205) (1 subtests) typ Typing notifications don't leak ban Non-present room members cannot ban others +ban Non-present room members cannot ban others (3 subtests) psh Getting push rules doesn't corrupt the cache SYN-390 +psh Getting push rules doesn't corrupt the cache SYN-390 (3 subtests) inv Test that we can be reinvited to a room we created syn Multiple calls to /sync should not cause 500 errors +syn Multiple calls to /sync should not cause 500 errors (6 subtests) gst Guest user can call /events on another world_readable room (SYN-606) gst Real user can call /events on another world_readable room (SYN-606) gst Events come down the correct room diff --git a/are-we-synapse-yet.py b/are-we-synapse-yet.py index 10b1be28a..8d551575f 100755 --- a/are-we-synapse-yet.py +++ b/are-we-synapse-yet.py @@ -3,7 +3,7 @@ from __future__ import division import argparse import re -import sys +import os # Usage: $ ./are-we-synapse-yet.py [-v] results.tap # This script scans a results.tap file from Dendrite's CI process and spits out @@ -156,6 +156,7 @@ def parse_test_line(line): # ✓ POST /register downcases capitals in usernames # ... def print_stats(header_name, gid_to_tests, gid_to_name, verbose): + ci = os.getenv("CI") # When running from GHA, this groups the subsections subsections = [] # Registration: 100% (13/13 tests) subsection_test_names = {} # 'subsection name': ["✓ Test 1", "✓ Test 2", "× Test 3"] total_passing = 0 @@ -169,7 +170,7 @@ def print_stats(header_name, gid_to_tests, gid_to_name, verbose): for name, passing in tests.items(): if passing: group_passing += 1 - test_names_and_marks.append(f"{'✓' if passing else '×'} {name}") + test_names_and_marks.append(f"{'✅' if passing else '❌'} {name}") total_tests += group_total total_passing += group_passing @@ -186,11 +187,11 @@ def print_stats(header_name, gid_to_tests, gid_to_name, verbose): print("%s: %s (%d/%d tests)" % (header_name, pct, total_passing, total_tests)) print("-" * (len(header_name)+1)) for line in subsections: - print(" %s" % (line,)) + print("%s%s" % ("::group::" if ci and verbose else "", line,)) if verbose: for test_name_and_pass_mark in subsection_test_names[line]: print(" %s" % (test_name_and_pass_mark,)) - print("") + print("%s" % ("::endgroup::" if ci else "")) print("") def main(results_tap_path, verbose): diff --git a/build.cmd b/build.cmd new file mode 100644 index 000000000..9e90622c8 --- /dev/null +++ b/build.cmd @@ -0,0 +1,51 @@ +@echo off + +:ENTRY_POINT + setlocal EnableDelayedExpansion + + REM script base dir + set SCRIPTDIR=%~dp0 + set PROJDIR=%SCRIPTDIR:~0,-1% + + REM Put installed packages into ./bin + set GOBIN=%PROJDIR%\bin + + set FLAGS= + + REM Check if sources are under Git control + if not exist ".git" goto :CHECK_BIN + + REM set BUILD=`git rev-parse --short HEAD \\ ""` + FOR /F "tokens=*" %%X IN ('git rev-parse --short HEAD') DO ( + set BUILD=%%X + ) + + REM set BRANCH=`(git symbolic-ref --short HEAD \ tr -d \/ ) \\ ""` + FOR /F "tokens=*" %%X IN ('git symbolic-ref --short HEAD') DO ( + set BRANCHRAW=%%X + set BRANCH=!BRANCHRAW:/=! + ) + if "%BRANCH%" == "main" set BRANCH= + + set FLAGS=-X github.com/matrix-org/dendrite/internal.branch=%BRANCH% -X github.com/matrix-org/dendrite/internal.build=%BUILD% + +:CHECK_BIN + if exist "bin" goto :ALL_SET + mkdir "bin" + +:ALL_SET + set CGO_ENABLED=1 + for /D %%P in (cmd\*) do ( + go build -trimpath -ldflags "%FLAGS%" -v -o ".\bin" ".\%%P" + ) + + set CGO_ENABLED=0 + set GOOS=js + set GOARCH=wasm + go build -trimpath -ldflags "%FLAGS%" -o bin\main.wasm .\cmd\dendritejs-pinecone + + goto :DONE + +:DONE + echo Done + endlocal \ No newline at end of file diff --git a/build/docker/config/dendrite.yaml b/build/docker/config/dendrite.yaml index c01ab62e7..e3a0316dc 100644 --- a/build/docker/config/dendrite.yaml +++ b/build/docker/config/dendrite.yaml @@ -62,6 +62,17 @@ global: - matrix.org - vector.im + # Disables federation. Dendrite will not be able to make any outbound HTTP requests + # to other servers and the federation API will not be exposed. + disable_federation: false + + # Configures the handling of presence events. + presence: + # Whether inbound presence events are allowed, e.g. receiving presence events from other servers + enable_inbound: false + # Whether outbound presence events are allowed, e.g. sending presence events to other servers + enable_outbound: false + # Configuration for NATS JetStream jetstream: # A list of NATS Server addresses to connect to. If none are specified, an @@ -160,12 +171,6 @@ client_api: threshold: 5 cooloff_ms: 500 -# Configuration for the EDU server. -edu_server: - internal_api: - listen: http://0.0.0.0:7778 - connect: http://edu_server:7778 - # Configuration for the Federation API. federation_api: internal_api: @@ -179,12 +184,6 @@ federation_api: max_idle_conns: 2 conn_max_lifetime: -1 - # List of paths to X.509 certificates to be used by the external federation listeners. - # These certificates will be used to calculate the TLS fingerprints and other servers - # will expect the certificate to match these fingerprints. Certificates must be in PEM - # format. - federation_certificates: [] - # How many times we will try to resend a failed transaction to a specific server. The # backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds etc. send_max_retries: 16 diff --git a/build/docker/docker-compose.polylith.yml b/build/docker/docker-compose.polylith.yml index 207d0451a..de0ab0aa2 100644 --- a/build/docker/docker-compose.polylith.yml +++ b/build/docker/docker-compose.polylith.yml @@ -84,18 +84,6 @@ services: - internal restart: unless-stopped - edu_server: - hostname: edu_server - image: matrixdotorg/dendrite-polylith:latest - command: eduserver - volumes: - - ./config:/etc/dendrite - depends_on: - - jetstream - networks: - - internal - restart: unless-stopped - federation_api: hostname: federation_api image: matrixdotorg/dendrite-polylith:latest diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 865457010..9cc94d650 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -1,3 +1,17 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package gobind import ( @@ -9,7 +23,6 @@ import ( "fmt" "io" "io/ioutil" - "log" "net" "net/http" "os" @@ -22,11 +35,9 @@ import ( "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/conn" "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms" + "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/users" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" - "github.com/matrix-org/dendrite/eduserver" - "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationapi" - "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/roomserver" @@ -41,6 +52,7 @@ import ( "golang.org/x/net/http2" "golang.org/x/net/http2/h2c" + pineconeConnections "github.com/matrix-org/pinecone/connections" pineconeMulticast "github.com/matrix-org/pinecone/multicast" pineconeRouter "github.com/matrix-org/pinecone/router" pineconeSessions "github.com/matrix-org/pinecone/sessions" @@ -60,11 +72,9 @@ type DendriteMonolith struct { PineconeRouter *pineconeRouter.Router PineconeMulticast *pineconeMulticast.Multicast PineconeQUIC *pineconeSessions.Sessions + PineconeManager *pineconeConnections.ConnectionManager StorageDirectory string CacheDirectory string - staticPeerURI string - staticPeerMutex sync.RWMutex - staticPeerAttempt chan struct{} listener net.Listener httpServer *http.Server processContext *process.ProcessContext @@ -80,7 +90,7 @@ func (m *DendriteMonolith) PeerCount(peertype int) int { } func (m *DendriteMonolith) SessionCount() int { - return len(m.PineconeQUIC.Sessions()) + return len(m.PineconeQUIC.Protocol("matrix").Sessions()) } func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) { @@ -93,15 +103,8 @@ func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) { } func (m *DendriteMonolith) SetStaticPeer(uri string) { - m.staticPeerMutex.Lock() - m.staticPeerURI = strings.TrimSpace(uri) - m.staticPeerMutex.Unlock() - m.DisconnectType(int(pineconeRouter.PeerTypeRemote)) - if uri != "" { - go func() { - m.staticPeerAttempt <- struct{}{} - }() - } + m.PineconeManager.RemovePeers() + m.PineconeManager.AddPeer(strings.TrimSpace(uri)) } func (m *DendriteMonolith) DisconnectType(peertype int) { @@ -199,43 +202,6 @@ func (m *DendriteMonolith) RegisterDevice(localpart, deviceID string) (string, e return loginRes.Device.AccessToken, nil } -func (m *DendriteMonolith) staticPeerConnect() { - connected := map[string]bool{} // URI -> connected? - attempt := func() { - m.staticPeerMutex.RLock() - uri := m.staticPeerURI - m.staticPeerMutex.RUnlock() - if uri == "" { - return - } - for k := range connected { - delete(connected, k) - } - for _, uri := range strings.Split(uri, ",") { - connected[strings.TrimSpace(uri)] = false - } - for _, info := range m.PineconeRouter.Peers() { - connected[info.URI] = true - } - for k, online := range connected { - if !online { - if err := conn.ConnectToPeer(m.PineconeRouter, k); err != nil { - logrus.WithError(err).Error("Failed to connect to static peer") - } - } - } - } - for { - select { - case <-m.processContext.Context().Done(): - case <-m.staticPeerAttempt: - attempt() - case <-time.After(time.Second * 5): - attempt() - } - } -} - // nolint:gocyclo func (m *DendriteMonolith) Start() { var err error @@ -259,7 +225,7 @@ func (m *DendriteMonolith) Start() { pk = sk.Public().(ed25519.PublicKey) } - m.listener, err = net.Listen("tcp", "localhost:65432") + m.listener, err = net.Listen("tcp", ":65432") if err != nil { panic(err) } @@ -270,10 +236,10 @@ func (m *DendriteMonolith) Start() { m.logger.SetOutput(BindLogger{}) logrus.SetOutput(BindLogger{}) - logger := log.New(os.Stdout, "PINECONE: ", 0) - m.PineconeRouter = pineconeRouter.NewRouter(logger, sk, false) - m.PineconeQUIC = pineconeSessions.NewSessions(logger, m.PineconeRouter) - m.PineconeMulticast = pineconeMulticast.NewMulticast(logger, m.PineconeRouter) + m.PineconeRouter = pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk, false) + m.PineconeQUIC = pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), m.PineconeRouter, []string{"matrix"}) + m.PineconeMulticast = pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), m.PineconeRouter) + m.PineconeManager = pineconeConnections.NewConnectionManager(m.PineconeRouter) prefix := hex.EncodeToString(pk) cfg := &config.Dendrite{} @@ -281,6 +247,7 @@ func (m *DendriteMonolith) Start() { cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk)) cfg.Global.PrivateKey = sk cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID) + cfg.Global.JetStream.InMemory = true cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/%s", m.StorageDirectory, prefix)) cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-account.db", m.StorageDirectory, prefix)) cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-mediaapi.db", m.StorageDirectory)) @@ -315,16 +282,15 @@ func (m *DendriteMonolith) Start() { m.userAPI = userapi.NewInternalAPI(base, accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, base.PushGatewayHTTPClient()) keyAPI.SetUserAPI(m.userAPI) - eduInputAPI := eduserver.NewInternalAPI( - base, cache.New(), m.userAPI, - ) - asAPI := appservice.NewInternalAPI(base, m.userAPI, rsAPI) // The underlying roomserver implementation needs to be able to call the fedsender. // This is different to rsAPI which can be the http client which doesn't need this dependency rsAPI.SetFederationAPI(fsAPI, keyRing) + userProvider := users.NewPineconeUserProvider(m.PineconeRouter, m.PineconeQUIC, m.userAPI, federation) + roomProvider := rooms.NewPineconeRoomProvider(m.PineconeRouter, m.PineconeQUIC, fsAPI, federation) + monolith := setup.Monolith{ Config: base.Cfg, AccountDB: accountDB, @@ -332,13 +298,13 @@ func (m *DendriteMonolith) Start() { FedClient: federation, KeyRing: keyRing, - AppserviceAPI: asAPI, - EDUInternalAPI: eduInputAPI, - FederationAPI: fsAPI, - RoomserverAPI: rsAPI, - UserAPI: m.userAPI, - KeyAPI: keyAPI, - ExtPublicRoomsProvider: rooms.NewPineconeRoomProvider(m.PineconeRouter, m.PineconeQUIC, fsAPI, federation), + AppserviceAPI: asAPI, + FederationAPI: fsAPI, + RoomserverAPI: rsAPI, + UserAPI: m.userAPI, + KeyAPI: keyAPI, + ExtPublicRoomsProvider: roomProvider, + ExtUserDirectoryProvider: userProvider, } monolith.AddAllPublicRoutes( base.ProcessContext, @@ -354,12 +320,15 @@ func (m *DendriteMonolith) Start() { httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux) httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux) httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux) + httpRouter.HandleFunc("/pinecone", m.PineconeRouter.ManholeHandler) pMux := mux.NewRouter().SkipClean(true).UseEncodedPath() + pMux.PathPrefix(users.PublicURL).HandlerFunc(userProvider.FederatedUserProfiles) pMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux) pMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux) - pHTTP := m.PineconeQUIC.HTTP() + pHTTP := m.PineconeQUIC.Protocol("matrix").HTTP() + pHTTP.Mux().Handle(users.PublicURL, pMux) pHTTP.Mux().Handle(httputil.PublicFederationPathPrefix, pMux) pHTTP.Mux().Handle(httputil.PublicMediaPathPrefix, pMux) @@ -379,33 +348,23 @@ func (m *DendriteMonolith) Start() { m.processContext = base.ProcessContext - m.staticPeerAttempt = make(chan struct{}, 1) - go m.staticPeerConnect() - go func() { m.logger.Info("Listening on ", cfg.Global.ServerName) - m.logger.Fatal(m.httpServer.Serve(m.PineconeQUIC)) + m.logger.Fatal(m.httpServer.Serve(m.PineconeQUIC.Protocol("matrix"))) }() go func() { logrus.Info("Listening on ", m.listener.Addr()) logrus.Fatal(http.Serve(m.listener, httpRouter)) }() - go func() { - logrus.Info("Sending wake-up message to known nodes") - req := &api.PerformBroadcastEDURequest{} - res := &api.PerformBroadcastEDUResponse{} - if err := fsAPI.PerformBroadcastEDU(context.TODO(), req, res); err != nil { - logrus.WithError(err).Error("Failed to send wake-up message to known nodes") - } - }() } func (m *DendriteMonolith) Stop() { + m.processContext.ShutdownDendrite() _ = m.listener.Close() m.PineconeMulticast.Stop() _ = m.PineconeQUIC.Close() - m.processContext.ShutdownDendrite() _ = m.PineconeRouter.Close() + m.processContext.WaitForComponentsToFinish() } const MaxFrameSize = types.MaxFrameSize diff --git a/build/gobind-pinecone/platform_ios.go b/build/gobind-pinecone/platform_ios.go index 802d7faca..a89ebfcd0 100644 --- a/build/gobind-pinecone/platform_ios.go +++ b/build/gobind-pinecone/platform_ios.go @@ -1,3 +1,17 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build ios // +build ios diff --git a/build/gobind-pinecone/platform_other.go b/build/gobind-pinecone/platform_other.go index 2e81e2f43..2793026b8 100644 --- a/build/gobind-pinecone/platform_other.go +++ b/build/gobind-pinecone/platform_other.go @@ -1,3 +1,17 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build !ios // +build !ios diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go index 3329485aa..87dcad2e8 100644 --- a/build/gobind-yggdrasil/monolith.go +++ b/build/gobind-yggdrasil/monolith.go @@ -13,8 +13,6 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggrooms" - "github.com/matrix-org/dendrite/eduserver" - "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/httputil" @@ -23,6 +21,7 @@ import ( "github.com/matrix-org/dendrite/setup" "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" @@ -36,6 +35,7 @@ type DendriteMonolith struct { StorageDirectory string listener net.Listener httpServer *http.Server + processContext *process.ProcessContext } func (m *DendriteMonolith) BaseURL() string { @@ -87,6 +87,7 @@ func (m *DendriteMonolith) Start() { cfg.Global.PrivateKey = ygg.PrivateKey() cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID) cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", m.StorageDirectory)) + cfg.Global.JetStream.InMemory = true cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-account.db", m.StorageDirectory)) cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-mediaapi.db", m.StorageDirectory)) cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-syncapi.db", m.StorageDirectory)) @@ -101,6 +102,7 @@ func (m *DendriteMonolith) Start() { } base := base.NewBaseDendrite(cfg, "Monolith") + m.processContext = base.ProcessContext defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() @@ -119,10 +121,6 @@ func (m *DendriteMonolith) Start() { userAPI := userapi.NewInternalAPI(base, accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, base.PushGatewayHTTPClient()) keyAPI.SetUserAPI(userAPI) - eduInputAPI := eduserver.NewInternalAPI( - base, cache.New(), userAPI, - ) - asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) @@ -137,12 +135,11 @@ func (m *DendriteMonolith) Start() { FedClient: federation, KeyRing: keyRing, - AppserviceAPI: asAPI, - EDUInternalAPI: eduInputAPI, - FederationAPI: fsAPI, - RoomserverAPI: rsAPI, - UserAPI: userAPI, - KeyAPI: keyAPI, + AppserviceAPI: asAPI, + FederationAPI: fsAPI, + RoomserverAPI: rsAPI, + UserAPI: userAPI, + KeyAPI: keyAPI, ExtPublicRoomsProvider: yggrooms.NewYggdrasilRoomProvider( ygg, fsAPI, federation, ), @@ -197,9 +194,12 @@ func (m *DendriteMonolith) Start() { }() } -func (m *DendriteMonolith) Suspend() { - m.logger.Info("Suspending monolith") +func (m *DendriteMonolith) Stop() { if err := m.httpServer.Close(); err != nil { m.logger.Warn("Error stopping HTTP server:", err) } + if m.processContext != nil { + m.processContext.ShutdownDendrite() + m.processContext.WaitForComponentsToFinish() + } } diff --git a/clientapi/auth/authtypes/profile.go b/clientapi/auth/authtypes/profile.go index 902850bc0..29468c168 100644 --- a/clientapi/auth/authtypes/profile.go +++ b/clientapi/auth/authtypes/profile.go @@ -17,6 +17,7 @@ package authtypes // Profile represents the profile for a Matrix account. type Profile struct { Localpart string `json:"local_part"` + ServerName string `json:"server_name,omitempty"` // NOTSPEC: only set by Pinecone user provider DisplayName string `json:"display_name"` AvatarURL string `json:"avatar_url"` } diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 31a53a706..e2f8d3f32 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -20,7 +20,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/api" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/clientapi/routing" - eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/transactions" keyserverAPI "github.com/matrix-org/dendrite/keyserver/api" @@ -40,26 +39,32 @@ func AddPublicRoutes( cfg *config.ClientAPI, federation *gomatrixserverlib.FederationClient, rsAPI roomserverAPI.RoomserverInternalAPI, - eduInputAPI eduServerAPI.EDUServerInputAPI, asAPI appserviceAPI.AppServiceQueryAPI, transactionsCache *transactions.Cache, fsAPI federationAPI.FederationInternalAPI, userAPI userapi.UserInternalAPI, + userDirectoryProvider userapi.UserDirectoryProvider, keyAPI keyserverAPI.KeyInternalAPI, extRoomsProvider api.ExtraPublicRoomsProvider, mscCfg *config.MSCs, ) { - js, _ := jetstream.Prepare(process, &cfg.Matrix.JetStream) + js, natsClient := jetstream.Prepare(process, &cfg.Matrix.JetStream) syncProducer := &producers.SyncAPIProducer{ - JetStream: js, - Topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputClientData), + JetStream: js, + TopicClientData: cfg.Matrix.JetStream.Prefixed(jetstream.OutputClientData), + TopicReceiptEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent), + TopicSendToDeviceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), + TopicTypingEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent), + TopicPresenceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent), + UserAPI: userAPI, + ServerName: cfg.Matrix.ServerName, } routing.Setup( - router, synapseAdminRouter, cfg, eduInputAPI, rsAPI, asAPI, - userAPI, federation, + router, synapseAdminRouter, cfg, rsAPI, asAPI, + userAPI, userDirectoryProvider, federation, syncProducer, transactionsCache, fsAPI, keyAPI, - extRoomsProvider, mscCfg, + extRoomsProvider, mscCfg, natsClient, ) } diff --git a/clientapi/producers/syncapi.go b/clientapi/producers/syncapi.go index 9ab90391d..187e3412d 100644 --- a/clientapi/producers/syncapi.go +++ b/clientapi/producers/syncapi.go @@ -15,32 +15,45 @@ package producers import ( + "context" "encoding/json" + "strconv" + "time" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/syncapi/types" + userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" "github.com/nats-io/nats.go" log "github.com/sirupsen/logrus" ) // SyncAPIProducer produces events for the sync API server to consume type SyncAPIProducer struct { - Topic string - JetStream nats.JetStreamContext + TopicClientData string + TopicReceiptEvent string + TopicSendToDeviceEvent string + TopicTypingEvent string + TopicPresenceEvent string + JetStream nats.JetStreamContext + ServerName gomatrixserverlib.ServerName + UserAPI userapi.UserInternalAPI } // SendData sends account data to the sync API server -func (p *SyncAPIProducer) SendData(userID string, roomID string, dataType string, readMarker *eventutil.ReadMarkerJSON) error { +func (p *SyncAPIProducer) SendData(userID string, roomID string, dataType string, readMarker *eventutil.ReadMarkerJSON, ignoredUsers *types.IgnoredUsers) error { m := &nats.Msg{ - Subject: p.Topic, + Subject: p.TopicClientData, Header: nats.Header{}, } m.Header.Set(jetstream.UserID, userID) data := eventutil.AccountData{ - RoomID: roomID, - Type: dataType, - ReadMarker: readMarker, + RoomID: roomID, + Type: dataType, + ReadMarker: readMarker, + IgnoredUsers: ignoredUsers, } var err error m.Data, err = json.Marshal(data) @@ -52,8 +65,130 @@ func (p *SyncAPIProducer) SendData(userID string, roomID string, dataType string "user_id": userID, "room_id": roomID, "data_type": dataType, - }).Tracef("Producing to topic '%s'", p.Topic) + }).Tracef("Producing to topic '%s'", p.TopicClientData) _, err = p.JetStream.PublishMsg(m) return err } + +func (p *SyncAPIProducer) SendReceipt( + ctx context.Context, + userID, roomID, eventID, receiptType string, timestamp gomatrixserverlib.Timestamp, +) error { + m := &nats.Msg{ + Subject: p.TopicReceiptEvent, + Header: nats.Header{}, + } + m.Header.Set(jetstream.UserID, userID) + m.Header.Set(jetstream.RoomID, roomID) + m.Header.Set(jetstream.EventID, eventID) + m.Header.Set("type", receiptType) + m.Header.Set("timestamp", strconv.Itoa(int(timestamp))) + + log.WithFields(log.Fields{}).Tracef("Producing to topic '%s'", p.TopicReceiptEvent) + _, err := p.JetStream.PublishMsg(m, nats.Context(ctx)) + return err +} + +func (p *SyncAPIProducer) SendToDevice( + ctx context.Context, sender, userID, deviceID, eventType string, + message interface{}, +) error { + devices := []string{} + _, domain, err := gomatrixserverlib.SplitID('@', userID) + if err != nil { + return err + } + + // If the event is targeted locally then we want to expand the wildcard + // out into individual device IDs so that we can send them to each respective + // device. If the event isn't targeted locally then we can't expand the + // wildcard as we don't know about the remote devices, so instead we leave it + // as-is, so that the federation sender can send it on with the wildcard intact. + if domain == p.ServerName && deviceID == "*" { + var res userapi.QueryDevicesResponse + err = p.UserAPI.QueryDevices(context.TODO(), &userapi.QueryDevicesRequest{ + UserID: userID, + }, &res) + if err != nil { + return err + } + for _, dev := range res.Devices { + devices = append(devices, dev.ID) + } + } else { + devices = append(devices, deviceID) + } + + js, err := json.Marshal(message) + if err != nil { + return err + } + + log.WithFields(log.Fields{ + "user_id": userID, + "num_devices": len(devices), + "type": eventType, + }).Tracef("Producing to topic '%s'", p.TopicSendToDeviceEvent) + for _, device := range devices { + ote := &types.OutputSendToDeviceEvent{ + UserID: userID, + DeviceID: device, + SendToDeviceEvent: gomatrixserverlib.SendToDeviceEvent{ + Sender: sender, + Type: eventType, + Content: js, + }, + } + + eventJSON, err := json.Marshal(ote) + if err != nil { + log.WithError(err).Error("sendToDevice failed json.Marshal") + return err + } + m := &nats.Msg{ + Subject: p.TopicSendToDeviceEvent, + Data: eventJSON, + Header: nats.Header{}, + } + m.Header.Set("sender", sender) + m.Header.Set(jetstream.UserID, userID) + if _, err = p.JetStream.PublishMsg(m, nats.Context(ctx)); err != nil { + log.WithError(err).Error("sendToDevice failed t.Producer.SendMessage") + return err + } + } + return nil +} + +func (p *SyncAPIProducer) SendTyping( + ctx context.Context, userID, roomID string, typing bool, timeoutMS int64, +) error { + m := &nats.Msg{ + Subject: p.TopicTypingEvent, + Header: nats.Header{}, + } + m.Header.Set(jetstream.UserID, userID) + m.Header.Set(jetstream.RoomID, roomID) + m.Header.Set("typing", strconv.FormatBool(typing)) + m.Header.Set("timeout_ms", strconv.Itoa(int(timeoutMS))) + + _, err := p.JetStream.PublishMsg(m, nats.Context(ctx)) + return err +} + +func (p *SyncAPIProducer) SendPresence( + ctx context.Context, userID string, presence types.Presence, statusMsg *string, +) error { + m := nats.NewMsg(p.TopicPresenceEvent) + m.Header.Set(jetstream.UserID, userID) + m.Header.Set("presence", presence.String()) + if statusMsg != nil { + m.Header.Set("status_msg", *statusMsg) + } + + m.Header.Set("last_active_ts", strconv.Itoa(int(gomatrixserverlib.AsTimestamp(time.Now())))) + + _, err := p.JetStream.PublishMsg(m, nats.Context(ctx)) + return err +} diff --git a/clientapi/routing/account_data.go b/clientapi/routing/account_data.go index d8e982690..d0dd3ab8d 100644 --- a/clientapi/routing/account_data.go +++ b/clientapi/routing/account_data.go @@ -23,9 +23,9 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" - eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/internal/eventutil" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" @@ -95,10 +95,10 @@ func SaveAccountData( } } - if dataType == "m.fully_read" { + if dataType == "m.fully_read" || dataType == "m.push_rules" { return util.JSONResponse{ Code: http.StatusForbidden, - JSON: jsonerror.Forbidden("Unable to set read marker"), + JSON: jsonerror.Forbidden(fmt.Sprintf("Unable to modify %q using this API", dataType)), } } @@ -127,8 +127,14 @@ func SaveAccountData( return util.ErrorResponse(err) } + var ignoredUsers *types.IgnoredUsers + if dataType == "m.ignored_user_list" { + ignoredUsers = &types.IgnoredUsers{} + _ = json.Unmarshal(body, ignoredUsers) + } + // TODO: user API should do this since it's account data - if err := syncProducer.SendData(userID, roomID, dataType, nil); err != nil { + if err := syncProducer.SendData(userID, roomID, dataType, nil, ignoredUsers); err != nil { util.GetLogger(req.Context()).WithError(err).Error("syncProducer.SendData failed") return jsonerror.InternalServerError() } @@ -146,7 +152,7 @@ type fullyReadEvent struct { // SaveReadMarker implements POST /rooms/{roomId}/read_markers func SaveReadMarker( req *http.Request, - userAPI api.UserInternalAPI, rsAPI roomserverAPI.RoomserverInternalAPI, eduAPI eduserverAPI.EDUServerInputAPI, + userAPI api.UserInternalAPI, rsAPI roomserverAPI.RoomserverInternalAPI, syncProducer *producers.SyncAPIProducer, device *api.Device, roomID string, ) util.JSONResponse { // Verify that the user is a member of this room @@ -185,14 +191,14 @@ func SaveReadMarker( return util.ErrorResponse(err) } - if err := syncProducer.SendData(device.UserID, roomID, "m.fully_read", &r); err != nil { + if err := syncProducer.SendData(device.UserID, roomID, "m.fully_read", &r, nil); err != nil { util.GetLogger(req.Context()).WithError(err).Error("syncProducer.SendData failed") return jsonerror.InternalServerError() } // Handle the read receipt that may be included in the read marker if r.Read != "" { - return SetReceipt(req, eduAPI, device, roomID, "m.read", r.Read) + return SetReceipt(req, syncProducer, device, roomID, "m.read", r.Read) } return util.JSONResponse{ diff --git a/clientapi/routing/presence.go b/clientapi/routing/presence.go new file mode 100644 index 000000000..093a62464 --- /dev/null +++ b/clientapi/routing/presence.go @@ -0,0 +1,137 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "fmt" + "net/http" + "strconv" + "time" + + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/clientapi/producers" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/nats-io/nats.go" + log "github.com/sirupsen/logrus" +) + +type presenceReq struct { + Presence string `json:"presence"` + StatusMsg *string `json:"status_msg,omitempty"` +} + +func SetPresence( + req *http.Request, + cfg *config.ClientAPI, + device *api.Device, + producer *producers.SyncAPIProducer, + userID string, +) util.JSONResponse { + if !cfg.Matrix.Presence.EnableOutbound { + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } + } + if device.UserID != userID { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("Unable to set presence for other user."), + } + } + var presence presenceReq + parseErr := httputil.UnmarshalJSONRequest(req, &presence) + if parseErr != nil { + return *parseErr + } + + presenceStatus, ok := types.PresenceFromString(presence.Presence) + if !ok { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.Unknown(fmt.Sprintf("Unknown presence '%s'.", presence.Presence)), + } + } + err := producer.SendPresence(req.Context(), userID, presenceStatus, presence.StatusMsg) + if err != nil { + log.WithError(err).Errorf("failed to update presence") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: jsonerror.InternalServerError(), + } + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } +} + +func GetPresence( + req *http.Request, + device *api.Device, + natsClient *nats.Conn, + presenceTopic string, + userID string, +) util.JSONResponse { + msg := nats.NewMsg(presenceTopic) + msg.Header.Set(jetstream.UserID, userID) + + presence, err := natsClient.RequestMsg(msg, time.Second*10) + if err != nil { + log.WithError(err).Errorf("unable to get presence") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: jsonerror.InternalServerError(), + } + } + + statusMsg := presence.Header.Get("status_msg") + e := presence.Header.Get("error") + if e != "" { + log.Errorf("received error msg from nats: %s", e) + return util.JSONResponse{ + Code: http.StatusOK, + JSON: types.PresenceClientResponse{ + Presence: types.PresenceUnavailable.String(), + }, + } + } + lastActive, err := strconv.Atoi(presence.Header.Get("last_active_ts")) + if err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: jsonerror.InternalServerError(), + } + } + + p := types.PresenceInternal{LastActiveTS: gomatrixserverlib.Timestamp(lastActive)} + currentlyActive := p.CurrentlyActive() + return util.JSONResponse{ + Code: http.StatusOK, + JSON: types.PresenceClientResponse{ + CurrentlyActive: ¤tlyActive, + LastActiveAgo: p.LastActiveAgo(), + Presence: presence.Header.Get("presence"), + StatusMsg: &statusMsg, + }, + } +} diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index dd1da1806..3f91b4c93 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -180,7 +180,7 @@ func SetAvatarURL( return jsonerror.InternalServerError() } - if err := api.SendEvents(req.Context(), rsAPI, api.KindNew, events, cfg.Matrix.ServerName, cfg.Matrix.ServerName, nil, false); err != nil { + if err := api.SendEvents(req.Context(), rsAPI, api.KindNew, events, cfg.Matrix.ServerName, cfg.Matrix.ServerName, nil, true); err != nil { util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed") return jsonerror.InternalServerError() } diff --git a/clientapi/routing/receipt.go b/clientapi/routing/receipt.go index fe8fe765d..0f9b1b4ff 100644 --- a/clientapi/routing/receipt.go +++ b/clientapi/routing/receipt.go @@ -19,21 +19,20 @@ import ( "net/http" "time" + "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/dendrite/eduserver/api" - userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" "github.com/sirupsen/logrus" ) -func SetReceipt(req *http.Request, eduAPI api.EDUServerInputAPI, device *userapi.Device, roomId, receiptType, eventId string) util.JSONResponse { +func SetReceipt(req *http.Request, syncProducer *producers.SyncAPIProducer, device *userapi.Device, roomID, receiptType, eventID string) util.JSONResponse { timestamp := gomatrixserverlib.AsTimestamp(time.Now()) logrus.WithFields(logrus.Fields{ - "roomId": roomId, + "roomID": roomID, "receiptType": receiptType, - "eventId": eventId, + "eventID": eventID, "userId": device.UserID, "timestamp": timestamp, }).Debug("Setting receipt") @@ -43,7 +42,7 @@ func SetReceipt(req *http.Request, eduAPI api.EDUServerInputAPI, device *userapi return util.MessageResponse(400, fmt.Sprintf("receipt type must be m.read not '%s'", receiptType)) } - if err := api.SendReceipt(req.Context(), eduAPI, device.UserID, roomId, eventId, receiptType, timestamp); err != nil { + if err := syncProducer.SendReceipt(req.Context(), device.UserID, roomID, eventID, receiptType, timestamp); err != nil { return util.ErrorResponse(err) } diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index af2e99ed0..8253f3155 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -31,6 +31,7 @@ import ( "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/setup/config" + "github.com/tidwall/gjson" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/tokens" @@ -63,11 +64,6 @@ const ( sessionIDLength = 24 ) -func init() { - // Register prometheus metrics. They must be registered to be exposed. - prometheus.MustRegister(amtRegUsers) -} - // sessionsDict keeps track of completed auth stages for each session. // It shouldn't be passed by value because it contains a mutex. type sessionsDict struct { @@ -525,22 +521,37 @@ func Register( userAPI userapi.UserRegisterAPI, cfg *config.ClientAPI, ) util.JSONResponse { + defer req.Body.Close() // nolint: errcheck + reqBody, err := ioutil.ReadAll(req.Body) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.NotJSON("Unable to read request body"), + } + } + var r registerRequest - resErr := httputil.UnmarshalJSONRequest(req, &r) - if resErr != nil { + sessionID := gjson.GetBytes(reqBody, "auth.session").String() + if sessionID == "" { + // Generate a new, random session ID + sessionID = util.RandomString(sessionIDLength) + } else if data, ok := sessions.getParams(sessionID); ok { + // Use the parameters from the session as our defaults. + // Some of these might end up being overwritten if the + // values are specified again in the request body. + r.Username = data.Username + r.Password = data.Password + r.DeviceID = data.DeviceID + r.InitialDisplayName = data.InitialDisplayName + r.InhibitLogin = data.InhibitLogin + } + if resErr := httputil.UnmarshalJSON(reqBody, &r); resErr != nil { return *resErr } if req.URL.Query().Get("kind") == "guest" { return handleGuestRegistration(req, r, cfg, userAPI) } - // Retrieve or generate the sessionID - sessionID := r.Auth.Session - if sessionID == "" { - // Generate a new, random session ID - sessionID = util.RandomString(sessionIDLength) - } - // Don't allow numeric usernames less than MAX_INT64. if _, err := strconv.ParseInt(r.Username, 10, 64); err == nil { return util.JSONResponse{ @@ -568,7 +579,7 @@ func Register( case r.Type == authtypes.LoginTypeApplicationService && accessTokenErr == nil: // Spec-compliant case (the access_token is specified and the login type // is correctly set, so it's an appservice registration) - if resErr = validateApplicationServiceUsername(r.Username); resErr != nil { + if resErr := validateApplicationServiceUsername(r.Username); resErr != nil { return *resErr } case accessTokenErr == nil: @@ -581,11 +592,11 @@ func Register( default: // Spec-compliant case (neither the access_token nor the login type are // specified, so it's a normal user registration) - if resErr = validateUsername(r.Username); resErr != nil { + if resErr := validateUsername(r.Username); resErr != nil { return *resErr } } - if resErr = validatePassword(r.Password); resErr != nil { + if resErr := validatePassword(r.Password); resErr != nil { return *resErr } @@ -835,24 +846,17 @@ func completeRegistration( } }() - if data, ok := sessions.getParams(sessionID); ok { - username = data.Username - password = data.Password - deviceID = data.DeviceID - displayName = data.InitialDisplayName - inhibitLogin = data.InhibitLogin - } if username == "" { return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON("missing username"), + JSON: jsonerror.MissingArgument("Missing username"), } } // Blank passwords are only allowed by registered application services if password == "" && appserviceID == "" { return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON("missing password"), + JSON: jsonerror.MissingArgument("Missing password"), } } var accRes userapi.PerformAccountCreationResponse diff --git a/clientapi/routing/room_tagging.go b/clientapi/routing/room_tagging.go index 83294b180..ce173613e 100644 --- a/clientapi/routing/room_tagging.go +++ b/clientapi/routing/room_tagging.go @@ -98,7 +98,7 @@ func PutTag( return jsonerror.InternalServerError() } - if err = syncProducer.SendData(userID, roomID, "m.tag", nil); err != nil { + if err = syncProducer.SendData(userID, roomID, "m.tag", nil, nil); err != nil { logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi") } @@ -151,7 +151,7 @@ func DeleteTag( } // TODO: user API should do this since it's account data - if err := syncProducer.SendData(userID, roomID, "m.tag", nil); err != nil { + if err := syncProducer.SendData(userID, roomID, "m.tag", nil, nil); err != nil { logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi") } diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index e173831ab..37d825b80 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -26,16 +26,18 @@ import ( clientutil "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" - eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/transactions" keyserverAPI "github.com/matrix-org/dendrite/keyserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/jetstream" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" + "github.com/nats-io/nats.go" + "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" ) @@ -47,18 +49,20 @@ import ( // nolint: gocyclo func Setup( publicAPIMux, synapseAdminRouter *mux.Router, cfg *config.ClientAPI, - eduAPI eduServerAPI.EDUServerInputAPI, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, userAPI userapi.UserInternalAPI, + userDirectoryProvider userapi.UserDirectoryProvider, federation *gomatrixserverlib.FederationClient, syncProducer *producers.SyncAPIProducer, transactionsCache *transactions.Cache, federationSender federationAPI.FederationInternalAPI, keyAPI keyserverAPI.KeyInternalAPI, extRoomsProvider api.ExtraPublicRoomsProvider, - mscCfg *config.MSCs, + mscCfg *config.MSCs, natsClient *nats.Conn, ) { + prometheus.MustRegister(amtRegUsers, sendEventDuration) + rateLimits := httputil.NewRateLimits(&cfg.RateLimiting) userInteractiveAuth := auth.NewUserInteractive(userAPI, cfg) @@ -466,7 +470,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SendTyping(req, device, vars["roomID"], vars["userID"], eduAPI, rsAPI) + return SendTyping(req, device, vars["roomID"], vars["userID"], rsAPI, syncProducer) }), ).Methods(http.MethodPut, http.MethodOptions) v3mux.Handle("/rooms/{roomID}/redact/{eventID}", @@ -495,7 +499,7 @@ func Setup( return util.ErrorResponse(err) } txnID := vars["txnID"] - return SendToDevice(req, device, eduAPI, transactionsCache, vars["eventType"], &txnID) + return SendToDevice(req, device, syncProducer, transactionsCache, vars["eventType"], &txnID) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -509,7 +513,7 @@ func Setup( return util.ErrorResponse(err) } txnID := vars["txnID"] - return SendToDevice(req, device, eduAPI, transactionsCache, vars["eventType"], &txnID) + return SendToDevice(req, device, syncProducer, transactionsCache, vars["eventType"], &txnID) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -780,20 +784,6 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) - // Element logs get flooded unless this is handled - v3mux.Handle("/presence/{userID}/status", - httputil.MakeExternalAPI("presence", func(req *http.Request) util.JSONResponse { - if r := rateLimits.Limit(req); r != nil { - return *r - } - // TODO: Set presence (probably the responsibility of a presence server not clientapi) - return util.JSONResponse{ - Code: http.StatusOK, - JSON: struct{}{}, - } - }), - ).Methods(http.MethodPut, http.MethodOptions) - v3mux.Handle("/voip/turnServer", httputil.MakeAuthAPI("turn_server", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { if r := rateLimits.Limit(req); r != nil { @@ -904,6 +894,7 @@ func Setup( device, userAPI, rsAPI, + userDirectoryProvider, cfg.Matrix.ServerName, postContent.SearchString, postContent.Limit, @@ -940,7 +931,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SaveReadMarker(req, userAPI, rsAPI, eduAPI, syncProducer, device, vars["roomID"]) + return SaveReadMarker(req, userAPI, rsAPI, syncProducer, device, vars["roomID"]) }), ).Methods(http.MethodPost, http.MethodOptions) @@ -957,6 +948,16 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) + v3mux.Handle("/rooms/{roomID}/upgrade", + httputil.MakeAuthAPI("rooms_upgrade", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return UpgradeRoom(req, device, cfg, vars["roomID"], userAPI, rsAPI, asAPI) + }), + ).Methods(http.MethodPost, http.MethodOptions) + v3mux.Handle("/devices", httputil.MakeAuthAPI("get_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return GetDevicesByLocalpart(req, userAPI, device) @@ -1295,7 +1296,25 @@ func Setup( return util.ErrorResponse(err) } - return SetReceipt(req, eduAPI, device, vars["roomId"], vars["receiptType"], vars["eventId"]) + return SetReceipt(req, syncProducer, device, vars["roomId"], vars["receiptType"], vars["eventId"]) }), ).Methods(http.MethodPost, http.MethodOptions) + v3mux.Handle("/presence/{userId}/status", + httputil.MakeAuthAPI("set_presence", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return SetPresence(req, cfg, device, syncProducer, vars["userId"]) + }), + ).Methods(http.MethodPut, http.MethodOptions) + v3mux.Handle("/presence/{userId}/status", + httputil.MakeAuthAPI("get_presence", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return GetPresence(req, device, natsClient, cfg.Matrix.JetStream.Prefixed(jetstream.RequestPresence), vars["userId"]) + }), + ).Methods(http.MethodGet, http.MethodOptions) } diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index 3d5993718..1211fa72d 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -46,10 +46,6 @@ var ( userRoomSendMutexes sync.Map // (roomID+userID) -> mutex. mutexes to ensure correct ordering of sendEvents ) -func init() { - prometheus.MustRegister(sendEventDuration) -} - var sendEventDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "dendrite", @@ -272,5 +268,24 @@ func generateSendEvent( JSON: jsonerror.Forbidden(err.Error()), // TODO: Is this error string comprehensible to the client? } } + + // User should not be able to send a tombstone event to the same room. + if e.Type() == "m.room.tombstone" { + content := make(map[string]interface{}) + if err = json.Unmarshal(e.Content(), &content); err != nil { + util.GetLogger(ctx).WithError(err).Error("Cannot unmarshal the event content.") + return nil, &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("Cannot unmarshal the event content."), + } + } + if content["replacement_room"] == e.RoomID() { + return nil, &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.InvalidParam("Cannot send tombstone event that points to the same room."), + } + } + } + return e.Event, nil } diff --git a/clientapi/routing/sendtodevice.go b/clientapi/routing/sendtodevice.go index 768e8e0e7..4a5f08883 100644 --- a/clientapi/routing/sendtodevice.go +++ b/clientapi/routing/sendtodevice.go @@ -18,17 +18,17 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/eduserver/api" + "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/internal/transactions" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" ) // SendToDevice handles PUT /_matrix/client/r0/sendToDevice/{eventType}/{txnId} -// sends the device events to the EDU Server +// sends the device events to the syncapi & federationsender func SendToDevice( req *http.Request, device *userapi.Device, - eduAPI api.EDUServerInputAPI, + syncProducer *producers.SyncAPIProducer, txnCache *transactions.Cache, eventType string, txnID *string, ) util.JSONResponse { @@ -48,8 +48,8 @@ func SendToDevice( for userID, byUser := range httpReq.Messages { for deviceID, message := range byUser { - if err := api.SendToDevice( - req.Context(), eduAPI, device.UserID, userID, deviceID, eventType, message, + if err := syncProducer.SendToDevice( + req.Context(), device.UserID, userID, deviceID, eventType, message, ); err != nil { util.GetLogger(req.Context()).WithError(err).Error("eduProducer.SendToDevice failed") return jsonerror.InternalServerError() diff --git a/clientapi/routing/sendtyping.go b/clientapi/routing/sendtyping.go index abd2061a8..6a27ee615 100644 --- a/clientapi/routing/sendtyping.go +++ b/clientapi/routing/sendtyping.go @@ -17,7 +17,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/eduserver/api" + "github.com/matrix-org/dendrite/clientapi/producers" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" @@ -32,9 +32,8 @@ type typingContentJSON struct { // sends the typing events to client API typingProducer func SendTyping( req *http.Request, device *userapi.Device, roomID string, - userID string, - eduAPI api.EDUServerInputAPI, - rsAPI roomserverAPI.RoomserverInternalAPI, + userID string, rsAPI roomserverAPI.RoomserverInternalAPI, + syncProducer *producers.SyncAPIProducer, ) util.JSONResponse { if device.UserID != userID { return util.JSONResponse{ @@ -56,9 +55,7 @@ func SendTyping( return *resErr } - if err := api.SendTyping( - req.Context(), eduAPI, userID, roomID, r.Typing, r.Timeout, - ); err != nil { + if err := syncProducer.SendTyping(req.Context(), userID, roomID, r.Typing, r.Timeout); err != nil { util.GetLogger(req.Context()).WithError(err).Error("eduProducer.Send failed") return jsonerror.InternalServerError() } diff --git a/clientapi/routing/upgrade_room.go b/clientapi/routing/upgrade_room.go new file mode 100644 index 000000000..00bde36b3 --- /dev/null +++ b/clientapi/routing/upgrade_room.go @@ -0,0 +1,92 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "net/http" + + appserviceAPI "github.com/matrix-org/dendrite/appservice/api" + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/clientapi/jsonerror" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/version" + "github.com/matrix-org/dendrite/setup/config" + userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" +) + +type upgradeRoomRequest struct { + NewVersion string `json:"new_version"` +} + +type upgradeRoomResponse struct { + ReplacementRoom string `json:"replacement_room"` +} + +// UpgradeRoom implements /upgrade +func UpgradeRoom( + req *http.Request, device *userapi.Device, + cfg *config.ClientAPI, + roomID string, profileAPI userapi.UserProfileAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, + asAPI appserviceAPI.AppServiceQueryAPI, +) util.JSONResponse { + var r upgradeRoomRequest + if rErr := httputil.UnmarshalJSONRequest(req, &r); rErr != nil { + return *rErr + } + + // Validate that the room version is supported + if _, err := version.SupportedRoomVersion(gomatrixserverlib.RoomVersion(r.NewVersion)); err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.UnsupportedRoomVersion("This server does not support that room version"), + } + } + + upgradeReq := roomserverAPI.PerformRoomUpgradeRequest{ + UserID: device.UserID, + RoomID: roomID, + RoomVersion: gomatrixserverlib.RoomVersion(r.NewVersion), + } + upgradeResp := roomserverAPI.PerformRoomUpgradeResponse{} + + rsAPI.PerformRoomUpgrade(req.Context(), &upgradeReq, &upgradeResp) + + if upgradeResp.Error != nil { + if upgradeResp.Error.Code == roomserverAPI.PerformErrorNoRoom { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("Room does not exist"), + } + } else if upgradeResp.Error.Code == roomserverAPI.PerformErrorNotAllowed { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden(upgradeResp.Error.Msg), + } + } else { + return jsonerror.InternalServerError() + } + + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: upgradeRoomResponse{ + ReplacementRoom: upgradeResp.NewRoomID, + }, + } +} diff --git a/clientapi/routing/userdirectory.go b/clientapi/routing/userdirectory.go index 2659bc9cc..ab73cf430 100644 --- a/clientapi/routing/userdirectory.go +++ b/clientapi/routing/userdirectory.go @@ -16,6 +16,7 @@ package routing import ( "context" + "database/sql" "fmt" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" @@ -35,6 +36,7 @@ func SearchUserDirectory( device *userapi.Device, userAPI userapi.UserInternalAPI, rsAPI api.RoomserverInternalAPI, + provider userapi.UserDirectoryProvider, serverName gomatrixserverlib.ServerName, searchString string, limit int, @@ -50,13 +52,12 @@ func SearchUserDirectory( } // First start searching local users. - userReq := &userapi.QuerySearchProfilesRequest{ SearchString: searchString, Limit: limit, } userRes := &userapi.QuerySearchProfilesResponse{} - if err := userAPI.QuerySearchProfiles(ctx, userReq, userRes); err != nil { + if err := provider.QuerySearchProfiles(ctx, userReq, userRes); err != nil { errRes := util.ErrorResponse(fmt.Errorf("userAPI.QuerySearchProfiles: %w", err)) return &errRes } @@ -67,7 +68,12 @@ func SearchUserDirectory( break } - userID := fmt.Sprintf("@%s:%s", user.Localpart, serverName) + var userID string + if user.ServerName != "" { + userID = fmt.Sprintf("@%s:%s", user.Localpart, user.ServerName) + } else { + userID = fmt.Sprintf("@%s:%s", user.Localpart, serverName) + } if _, ok := results[userID]; !ok { results[userID] = authtypes.FullyQualifiedProfile{ UserID: userID, @@ -87,7 +93,7 @@ func SearchUserDirectory( Limit: limit - len(results), } stateRes := &api.QueryKnownUsersResponse{} - if err := rsAPI.QueryKnownUsers(ctx, stateReq, stateRes); err != nil { + if err := rsAPI.QueryKnownUsers(ctx, stateReq, stateRes); err != nil && err != sql.ErrNoRows { errRes := util.ErrorResponse(fmt.Errorf("rsAPI.QueryKnownUsers: %w", err)) return &errRes } diff --git a/cmd/client-api-proxy/main.go b/cmd/client-api-proxy/main.go deleted file mode 100644 index 742ec3e31..000000000 --- a/cmd/client-api-proxy/main.go +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "flag" - "fmt" - "net/http" - "net/http/httputil" - "net/url" - "os" - "strings" - "time" - - log "github.com/sirupsen/logrus" -) - -const usage = `Usage: %s - -Create a single endpoint URL which clients can be pointed at. - -The client-server API in Dendrite is split across multiple processes -which listen on multiple ports. You cannot point a Matrix client at -any of those ports, as there will be unimplemented functionality. -In addition, all client-server API processes start with the additional -path prefix '/api', which Matrix clients will be unaware of. - -This tool will proxy requests for all client-server URLs and forward -them to their respective process. It will also add the '/api' path -prefix to incoming requests. - -THIS TOOL IS FOR TESTING AND NOT INTENDED FOR PRODUCTION USE. - -Arguments: - -` - -var ( - syncServerURL = flag.String("sync-api-server-url", "", "The base URL of the listening 'dendrite-sync-api-server' process. E.g. 'http://localhost:4200'") - clientAPIURL = flag.String("client-api-server-url", "", "The base URL of the listening 'dendrite-client-api-server' process. E.g. 'http://localhost:4321'") - mediaAPIURL = flag.String("media-api-server-url", "", "The base URL of the listening 'dendrite-media-api-server' process. E.g. 'http://localhost:7779'") - bindAddress = flag.String("bind-address", ":8008", "The listening port for the proxy.") - certFile = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS") - keyFile = flag.String("tls-key", "", "The PEM private key to use for TLS") -) - -func makeProxy(targetURL string) (*httputil.ReverseProxy, error) { - targetURL = strings.TrimSuffix(targetURL, "/") - - // Check that we can parse the URL. - _, err := url.Parse(targetURL) - if err != nil { - return nil, err - } - return &httputil.ReverseProxy{ - Director: func(req *http.Request) { - // URL.Path() removes the % escaping from the path. - // The % encoding will be added back when the url is encoded - // when the request is forwarded. - // This means that we will lose any unessecary escaping from the URL. - // Pratically this means that any distinction between '%2F' and '/' - // in the URL will be lost by the time it reaches the target. - path := req.URL.Path - log.WithFields(log.Fields{ - "path": path, - "url": targetURL, - "method": req.Method, - }).Print("proxying request") - newURL, err := url.Parse(targetURL) - // Set the path separately as we need to preserve '#' characters - // that would otherwise be interpreted as being the start of a URL - // fragment. - newURL.Path += path - if err != nil { - // We already checked that we can parse the URL - // So this shouldn't ever get hit. - panic(err) - } - // Copy the query parameters from the request. - newURL.RawQuery = req.URL.RawQuery - req.URL = newURL - }, - }, nil -} - -func main() { - flag.Usage = func() { - fmt.Fprintf(os.Stderr, usage, os.Args[0]) - flag.PrintDefaults() - } - - flag.Parse() - - if *syncServerURL == "" { - flag.Usage() - fmt.Fprintln(os.Stderr, "no --sync-api-server-url specified.") - os.Exit(1) - } - - if *clientAPIURL == "" { - flag.Usage() - fmt.Fprintln(os.Stderr, "no --client-api-server-url specified.") - os.Exit(1) - } - - if *mediaAPIURL == "" { - flag.Usage() - fmt.Fprintln(os.Stderr, "no --media-api-server-url specified.") - os.Exit(1) - } - syncProxy, err := makeProxy(*syncServerURL) - if err != nil { - panic(err) - } - clientProxy, err := makeProxy(*clientAPIURL) - if err != nil { - panic(err) - } - mediaProxy, err := makeProxy(*mediaAPIURL) - if err != nil { - panic(err) - } - - http.Handle("/_matrix/client/r0/sync", syncProxy) - http.Handle("/_matrix/media/v1/", mediaProxy) - http.Handle("/", clientProxy) - - srv := &http.Server{ - Addr: *bindAddress, - ReadTimeout: 1 * time.Minute, // how long we wait for the client to send the entire request (after connection accept) - WriteTimeout: 5 * time.Minute, // how long the proxy has to write the full response - } - - fmt.Println("Proxying requests to:") - fmt.Println(" /_matrix/client/r0/sync => ", *syncServerURL+"/api/_matrix/client/r0/sync") - fmt.Println(" /_matrix/media/v1 => ", *mediaAPIURL+"/api/_matrix/media/v1") - fmt.Println(" /* => ", *clientAPIURL+"/api/*") - fmt.Println("Listening on ", *bindAddress) - if *certFile != "" && *keyFile != "" { - panic(srv.ListenAndServeTLS(*certFile, *keyFile)) - } else { - panic(srv.ListenAndServe()) - } -} diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 8ce641914..26c8eb85f 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -29,7 +29,6 @@ import ( p2pdisc "github.com/libp2p/go-libp2p/p2p/discovery" "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/embed" - "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/keyserver" @@ -40,8 +39,6 @@ import ( "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/dendrite/eduserver/cache" - "github.com/sirupsen/logrus" _ "github.com/mattn/go-sqlite3" @@ -152,9 +149,6 @@ func main() { userAPI := userapi.NewInternalAPI(&base.Base, accountDB, &cfg.UserAPI, nil, keyAPI, rsAPI, base.Base.PushGatewayHTTPClient()) keyAPI.SetUserAPI(userAPI) - eduInputAPI := eduserver.NewInternalAPI( - &base.Base, cache.New(), userAPI, - ) asAPI := appservice.NewInternalAPI(&base.Base, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) fsAPI := federationapi.NewInternalAPI( @@ -180,7 +174,6 @@ func main() { KeyRing: keyRing, AppserviceAPI: asAPI, - EDUInternalAPI: eduInputAPI, FederationAPI: fsAPI, RoomserverAPI: rsAPI, UserAPI: userAPI, diff --git a/cmd/dendrite-demo-pinecone/conn/client.go b/cmd/dendrite-demo-pinecone/conn/client.go index e3cc0468c..27e18c2a3 100644 --- a/cmd/dendrite-demo-pinecone/conn/client.go +++ b/cmd/dendrite-demo-pinecone/conn/client.go @@ -1,3 +1,17 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package conn import ( @@ -53,21 +67,22 @@ func (y *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { } func createTransport(s *pineconeSessions.Sessions) *http.Transport { + proto := s.Protocol("matrix") tr := &http.Transport{ DisableKeepAlives: false, - Dial: s.Dial, - DialContext: s.DialContext, - DialTLS: s.DialTLS, - DialTLSContext: s.DialTLSContext, + Dial: proto.Dial, + DialContext: proto.DialContext, + DialTLS: proto.DialTLS, + DialTLSContext: proto.DialTLSContext, } tr.RegisterProtocol( "matrix", &RoundTripper{ inner: &http.Transport{ DisableKeepAlives: false, - Dial: s.Dial, - DialContext: s.DialContext, - DialTLS: s.DialTLS, - DialTLSContext: s.DialTLSContext, + Dial: proto.Dial, + DialContext: proto.DialContext, + DialTLS: proto.DialTLS, + DialTLSContext: proto.DialTLSContext, }, }, ) diff --git a/cmd/dendrite-demo-pinecone/conn/ws.go b/cmd/dendrite-demo-pinecone/conn/ws.go index ef403e290..ed85abd51 100644 --- a/cmd/dendrite-demo-pinecone/conn/ws.go +++ b/cmd/dendrite-demo-pinecone/conn/ws.go @@ -1,3 +1,17 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package conn import ( diff --git a/cmd/dendrite-demo-pinecone/defaults/defaults.go b/cmd/dendrite-demo-pinecone/defaults/defaults.go new file mode 100644 index 000000000..c92493137 --- /dev/null +++ b/cmd/dendrite-demo-pinecone/defaults/defaults.go @@ -0,0 +1,21 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package defaults + +import "github.com/matrix-org/gomatrixserverlib" + +var DefaultServerNames = map[gomatrixserverlib.ServerName]struct{}{ + "3bf0258d23c60952639cc4c69c71d1508a7d43a0475d9000ff900a1848411ec7": {}, +} diff --git a/cmd/dendrite-demo-pinecone/embed/embed_elementweb.go b/cmd/dendrite-demo-pinecone/embed/embed_elementweb.go index 8b3be72c1..d37362e21 100644 --- a/cmd/dendrite-demo-pinecone/embed/embed_elementweb.go +++ b/cmd/dendrite-demo-pinecone/embed/embed_elementweb.go @@ -1,3 +1,17 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build elementweb // +build elementweb diff --git a/cmd/dendrite-demo-pinecone/embed/embed_other.go b/cmd/dendrite-demo-pinecone/embed/embed_other.go index a4b223452..94360fce6 100644 --- a/cmd/dendrite-demo-pinecone/embed/embed_other.go +++ b/cmd/dendrite-demo-pinecone/embed/embed_other.go @@ -1,3 +1,17 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build !elementweb // +build !elementweb diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index 45f186985..dd1ab3697 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. +// Copyright 2022 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -22,11 +22,9 @@ import ( "flag" "fmt" "io/ioutil" - "log" "net" "net/http" "os" - "strings" "time" "github.com/gorilla/mux" @@ -35,11 +33,9 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/conn" "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/embed" "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms" + "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/users" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" - "github.com/matrix-org/dendrite/eduserver" - "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationapi" - "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/keyserver" @@ -50,6 +46,7 @@ import ( "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/gomatrixserverlib" + pineconeConnections "github.com/matrix-org/pinecone/connections" pineconeMulticast "github.com/matrix-org/pinecone/multicast" pineconeRouter "github.com/matrix-org/pinecone/router" pineconeSessions "github.com/matrix-org/pinecone/sessions" @@ -92,8 +89,14 @@ func main() { pk = sk.Public().(ed25519.PublicKey) } - logger := log.New(os.Stdout, "", 0) - pRouter := pineconeRouter.NewRouter(logger, sk, false) + pRouter := pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk, false) + pQUIC := pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), pRouter, []string{"matrix"}) + pMulticast := pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), pRouter) + pManager := pineconeConnections.NewConnectionManager(pRouter) + pMulticast.Start() + if instancePeer != nil && *instancePeer != "" { + pManager.AddPeer(*instancePeer) + } go func() { listener, err := net.Listen("tcp", *instanceListen) @@ -123,36 +126,6 @@ func main() { } }() - pQUIC := pineconeSessions.NewSessions(logger, pRouter) - pMulticast := pineconeMulticast.NewMulticast(logger, pRouter) - pMulticast.Start() - - connectToStaticPeer := func() { - connected := map[string]bool{} // URI -> connected? - for _, uri := range strings.Split(*instancePeer, ",") { - connected[strings.TrimSpace(uri)] = false - } - attempt := func() { - for k := range connected { - connected[k] = false - } - for _, info := range pRouter.Peers() { - connected[info.URI] = true - } - for k, online := range connected { - if !online { - if err := conn.ConnectToPeer(pRouter, k); err != nil { - logrus.WithError(err).Error("Failed to connect to static peer") - } - } - } - } - for { - attempt() - time.Sleep(time.Second * 5) - } - } - cfg := &config.Dendrite{} cfg.Defaults(true) cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk)) @@ -190,14 +163,13 @@ func main() { userAPI := userapi.NewInternalAPI(base, accountDB, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient()) keyAPI.SetUserAPI(userAPI) - eduInputAPI := eduserver.NewInternalAPI( - base, cache.New(), userAPI, - ) - asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) rsComponent.SetFederationAPI(fsAPI, keyRing) + userProvider := users.NewPineconeUserProvider(pRouter, pQUIC, userAPI, federation) + roomProvider := rooms.NewPineconeRoomProvider(pRouter, pQUIC, fsAPI, federation) + monolith := setup.Monolith{ Config: base.Cfg, AccountDB: accountDB, @@ -205,13 +177,13 @@ func main() { FedClient: federation, KeyRing: keyRing, - AppserviceAPI: asAPI, - EDUInternalAPI: eduInputAPI, - FederationAPI: fsAPI, - RoomserverAPI: rsAPI, - UserAPI: userAPI, - KeyAPI: keyAPI, - ExtPublicRoomsProvider: rooms.NewPineconeRoomProvider(pRouter, pQUIC, fsAPI, federation), + AppserviceAPI: asAPI, + FederationAPI: fsAPI, + RoomserverAPI: rsAPI, + UserAPI: userAPI, + KeyAPI: keyAPI, + ExtPublicRoomsProvider: roomProvider, + ExtUserDirectoryProvider: userProvider, } monolith.AddAllPublicRoutes( base.ProcessContext, @@ -247,13 +219,16 @@ func main() { logrus.WithError(err).Error("Failed to connect WebSocket peer to Pinecone switch") } }) + httpRouter.HandleFunc("/pinecone", pRouter.ManholeHandler) embed.Embed(httpRouter, *instancePort, "Pinecone Demo") pMux := mux.NewRouter().SkipClean(true).UseEncodedPath() + pMux.PathPrefix(users.PublicURL).HandlerFunc(userProvider.FederatedUserProfiles) pMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux) pMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux) - pHTTP := pQUIC.HTTP() + pHTTP := pQUIC.Protocol("matrix").HTTP() + pHTTP.Mux().Handle(users.PublicURL, pMux) pHTTP.Mux().Handle(httputil.PublicFederationPathPrefix, pMux) pHTTP.Mux().Handle(httputil.PublicMediaPathPrefix, pMux) @@ -270,25 +245,16 @@ func main() { Handler: pMux, } - go connectToStaticPeer() go func() { pubkey := pRouter.PublicKey() logrus.Info("Listening on ", hex.EncodeToString(pubkey[:])) - logrus.Fatal(httpServer.Serve(pQUIC)) + logrus.Fatal(httpServer.Serve(pQUIC.Protocol("matrix"))) }() go func() { httpBindAddr := fmt.Sprintf(":%d", *instancePort) logrus.Info("Listening on ", httpBindAddr) logrus.Fatal(http.ListenAndServe(httpBindAddr, httpRouter)) }() - go func() { - logrus.Info("Sending wake-up message to known nodes") - req := &api.PerformBroadcastEDURequest{} - res := &api.PerformBroadcastEDUResponse{} - if err := fsAPI.PerformBroadcastEDU(context.TODO(), req, res); err != nil { - logrus.WithError(err).Error("Failed to send wake-up message to known nodes") - } - }() base.WaitForShutdown() } diff --git a/cmd/dendrite-demo-pinecone/rooms/rooms.go b/cmd/dendrite-demo-pinecone/rooms/rooms.go index 5972d129f..0fafbedc3 100644 --- a/cmd/dendrite-demo-pinecone/rooms/rooms.go +++ b/cmd/dendrite-demo-pinecone/rooms/rooms.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. +// Copyright 2022 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ import ( "sync" "time" + "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/defaults" "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -50,9 +51,12 @@ func NewPineconeRoomProvider( } func (p *PineconeRoomProvider) Rooms() []gomatrixserverlib.PublicRoom { - list := []gomatrixserverlib.ServerName{} + list := map[gomatrixserverlib.ServerName]struct{}{} + for k := range defaults.DefaultServerNames { + list[k] = struct{}{} + } for _, k := range p.r.Peers() { - list = append(list, gomatrixserverlib.ServerName(k.PublicKey)) + list[gomatrixserverlib.ServerName(k.PublicKey)] = struct{}{} } return bulkFetchPublicRoomsFromServers(context.Background(), p.fedClient, list) } @@ -61,7 +65,7 @@ func (p *PineconeRoomProvider) Rooms() []gomatrixserverlib.PublicRoom { // Returns a list of public rooms. func bulkFetchPublicRoomsFromServers( ctx context.Context, fedClient *gomatrixserverlib.FederationClient, - homeservers []gomatrixserverlib.ServerName, + homeservers map[gomatrixserverlib.ServerName]struct{}, ) (publicRooms []gomatrixserverlib.PublicRoom) { limit := 200 // follow pipeline semantics, see https://blog.golang.org/pipelines for more info. @@ -74,7 +78,7 @@ func bulkFetchPublicRoomsFromServers( wg.Add(len(homeservers)) // concurrently query for public rooms reqctx, reqcancel := context.WithTimeout(ctx, time.Second*5) - for _, hs := range homeservers { + for hs := range homeservers { go func(homeserverDomain gomatrixserverlib.ServerName) { defer wg.Done() util.GetLogger(reqctx).WithField("hs", homeserverDomain).Info("Querying HS for public rooms") diff --git a/cmd/dendrite-demo-pinecone/users/users.go b/cmd/dendrite-demo-pinecone/users/users.go new file mode 100644 index 000000000..ebfb5cbe3 --- /dev/null +++ b/cmd/dendrite-demo-pinecone/users/users.go @@ -0,0 +1,163 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package users + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "sync" + "time" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + clienthttputil "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/defaults" + userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + + pineconeRouter "github.com/matrix-org/pinecone/router" + pineconeSessions "github.com/matrix-org/pinecone/sessions" +) + +type PineconeUserProvider struct { + r *pineconeRouter.Router + s *pineconeSessions.Sessions + userAPI userapi.UserProfileAPI + fedClient *gomatrixserverlib.FederationClient +} + +const PublicURL = "/_matrix/p2p/profiles" + +func NewPineconeUserProvider( + r *pineconeRouter.Router, + s *pineconeSessions.Sessions, + userAPI userapi.UserProfileAPI, + fedClient *gomatrixserverlib.FederationClient, +) *PineconeUserProvider { + p := &PineconeUserProvider{ + r: r, + s: s, + userAPI: userAPI, + fedClient: fedClient, + } + return p +} + +func (p *PineconeUserProvider) FederatedUserProfiles(w http.ResponseWriter, r *http.Request) { + req := &userapi.QuerySearchProfilesRequest{Limit: 25} + res := &userapi.QuerySearchProfilesResponse{} + if err := clienthttputil.UnmarshalJSONRequest(r, &req); err != nil { + w.WriteHeader(400) + return + } + if err := p.userAPI.QuerySearchProfiles(r.Context(), req, res); err != nil { + w.WriteHeader(400) + return + } + j, err := json.Marshal(res) + if err != nil { + w.WriteHeader(400) + return + } + w.WriteHeader(200) + _, _ = w.Write(j) +} + +func (p *PineconeUserProvider) QuerySearchProfiles(ctx context.Context, req *userapi.QuerySearchProfilesRequest, res *userapi.QuerySearchProfilesResponse) error { + list := map[gomatrixserverlib.ServerName]struct{}{} + for k := range defaults.DefaultServerNames { + list[k] = struct{}{} + } + for _, k := range p.r.Peers() { + list[gomatrixserverlib.ServerName(k.PublicKey)] = struct{}{} + } + res.Profiles = bulkFetchUserDirectoriesFromServers(context.Background(), req, p.fedClient, list) + return nil +} + +// bulkFetchUserDirectoriesFromServers fetches users from the list of homeservers. +// Returns a list of user profiles. +func bulkFetchUserDirectoriesFromServers( + ctx context.Context, req *userapi.QuerySearchProfilesRequest, + fedClient *gomatrixserverlib.FederationClient, + homeservers map[gomatrixserverlib.ServerName]struct{}, +) (profiles []authtypes.Profile) { + jsonBody, err := json.Marshal(req) + if err != nil { + return nil + } + + limit := 200 + // follow pipeline semantics, see https://blog.golang.org/pipelines for more info. + // goroutines send rooms to this channel + profileCh := make(chan authtypes.Profile, int(limit)) + // signalling channel to tell goroutines to stop sending rooms and quit + done := make(chan bool) + // signalling to say when we can close the room channel + var wg sync.WaitGroup + wg.Add(len(homeservers)) + // concurrently query for public rooms + reqctx, reqcancel := context.WithTimeout(ctx, time.Second*5) + for hs := range homeservers { + go func(homeserverDomain gomatrixserverlib.ServerName) { + defer wg.Done() + util.GetLogger(reqctx).WithField("hs", homeserverDomain).Info("Querying HS for users") + + jsonBodyReader := bytes.NewBuffer(jsonBody) + httpReq, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("matrix://%s%s", homeserverDomain, PublicURL), jsonBodyReader) + if err != nil { + util.GetLogger(reqctx).WithError(err).WithField("hs", homeserverDomain).Warn( + "bulkFetchUserDirectoriesFromServers: failed to create request", + ) + } + res := &userapi.QuerySearchProfilesResponse{} + if err = fedClient.DoRequestAndParseResponse(reqctx, httpReq, res); err != nil { + util.GetLogger(reqctx).WithError(err).WithField("hs", homeserverDomain).Warn( + "bulkFetchUserDirectoriesFromServers: failed to query hs", + ) + return + } + for _, profile := range res.Profiles { + profile.ServerName = string(homeserverDomain) + // atomically send a room or stop + select { + case profileCh <- profile: + case <-done: + case <-reqctx.Done(): + util.GetLogger(reqctx).WithError(err).WithField("hs", homeserverDomain).Info("Interrupted whilst sending profiles") + return + } + } + }(hs) + } + + select { + case <-time.After(5 * time.Second): + default: + wg.Wait() + } + reqcancel() + close(done) + close(profileCh) + + for profile := range profileCh { + profiles = append(profiles, profile) + } + + return profiles +} diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index b7e30ba2e..b840eb2b8 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -32,8 +32,6 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggrooms" - "github.com/matrix-org/dendrite/eduserver" - "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal" @@ -120,10 +118,6 @@ func main() { userAPI := userapi.NewInternalAPI(base, accountDB, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient()) keyAPI.SetUserAPI(userAPI) - eduInputAPI := eduserver.NewInternalAPI( - base, cache.New(), userAPI, - ) - asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) fsAPI := federationapi.NewInternalAPI( @@ -139,12 +133,11 @@ func main() { FedClient: federation, KeyRing: keyRing, - AppserviceAPI: asAPI, - EDUInternalAPI: eduInputAPI, - FederationAPI: fsAPI, - RoomserverAPI: rsAPI, - UserAPI: userAPI, - KeyAPI: keyAPI, + AppserviceAPI: asAPI, + FederationAPI: fsAPI, + RoomserverAPI: rsAPI, + UserAPI: userAPI, + KeyAPI: keyAPI, ExtPublicRoomsProvider: yggrooms.NewYggdrasilRoomProvider( ygg, fsAPI, federation, ), diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 3b952504b..1443ab5b1 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -19,8 +19,6 @@ import ( "os" "github.com/matrix-org/dendrite/appservice" - "github.com/matrix-org/dendrite/eduserver" - "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/roomserver" @@ -61,7 +59,6 @@ func main() { // itself. cfg.AppServiceAPI.InternalAPI.Connect = httpAPIAddr cfg.ClientAPI.InternalAPI.Connect = httpAPIAddr - cfg.EDUServer.InternalAPI.Connect = httpAPIAddr cfg.FederationAPI.InternalAPI.Connect = httpAPIAddr cfg.KeyServer.InternalAPI.Connect = httpAPIAddr cfg.MediaAPI.InternalAPI.Connect = httpAPIAddr @@ -136,14 +133,6 @@ func main() { rsImpl.SetUserAPI(userAPI) keyImpl.SetUserAPI(userAPI) - eduInputAPI := eduserver.NewInternalAPI( - base, cache.New(), userAPI, - ) - if base.UseHTTPAPIs { - eduserver.AddInternalRoutes(base.InternalAPIMux, eduInputAPI) - eduInputAPI = base.EDUServerClient() - } - monolith := setup.Monolith{ Config: base.Cfg, AccountDB: accountDB, @@ -151,12 +140,10 @@ func main() { FedClient: federation, KeyRing: keyRing, - AppserviceAPI: asAPI, - EDUInternalAPI: eduInputAPI, - FederationAPI: fsAPI, - RoomserverAPI: rsAPI, - UserAPI: userAPI, - KeyAPI: keyAPI, + AppserviceAPI: asAPI, FederationAPI: fsAPI, + RoomserverAPI: rsAPI, + UserAPI: userAPI, + KeyAPI: keyAPI, } monolith.AddAllPublicRoutes( base.ProcessContext, diff --git a/cmd/dendrite-polylith-multi/main.go b/cmd/dendrite-polylith-multi/main.go index edfe6cdb0..6226cc328 100644 --- a/cmd/dendrite-polylith-multi/main.go +++ b/cmd/dendrite-polylith-multi/main.go @@ -43,7 +43,6 @@ func main() { components := map[string]entrypoint{ "appservice": personalities.Appservice, "clientapi": personalities.ClientAPI, - "eduserver": personalities.EDUServer, "federationapi": personalities.FederationAPI, "keyserver": personalities.KeyServer, "mediaapi": personalities.MediaAPI, diff --git a/cmd/dendrite-polylith-multi/personalities/clientapi.go b/cmd/dendrite-polylith-multi/personalities/clientapi.go index a2036de35..1e509f88a 100644 --- a/cmd/dendrite-polylith-multi/personalities/clientapi.go +++ b/cmd/dendrite-polylith-multi/personalities/clientapi.go @@ -27,13 +27,12 @@ func ClientAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { asQuery := base.AppserviceHTTPClient() rsAPI := base.RoomserverHTTPClient() fsAPI := base.FederationAPIHTTPClient() - eduInputAPI := base.EDUServerClient() userAPI := base.UserAPIClient() keyAPI := base.KeyServerHTTPClient() clientapi.AddPublicRoutes( base.ProcessContext, base.PublicClientAPIMux, base.SynapseAdminMux, &base.Cfg.ClientAPI, - federation, rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, userAPI, + federation, rsAPI, asQuery, transactions.New(), fsAPI, userAPI, userAPI, keyAPI, nil, &cfg.MSCs, ) diff --git a/cmd/dendrite-polylith-multi/personalities/eduserver.go b/cmd/dendrite-polylith-multi/personalities/eduserver.go deleted file mode 100644 index 8719facb3..000000000 --- a/cmd/dendrite-polylith-multi/personalities/eduserver.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package personalities - -import ( - "github.com/matrix-org/dendrite/eduserver" - "github.com/matrix-org/dendrite/eduserver/cache" - basepkg "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" -) - -func EDUServer(base *basepkg.BaseDendrite, cfg *config.Dendrite) { - intAPI := eduserver.NewInternalAPI(base, cache.New(), base.UserAPIClient()) - eduserver.AddInternalRoutes(base.InternalAPIMux, intAPI) - - base.SetupAndServeHTTP( - base.Cfg.EDUServer.InternalAPI.Listen, // internal listener - basepkg.NoListener, // external listener - nil, nil, - ) -} diff --git a/cmd/dendrite-polylith-multi/personalities/federationapi.go b/cmd/dendrite-polylith-multi/personalities/federationapi.go index 44357d660..b82577ce3 100644 --- a/cmd/dendrite-polylith-multi/personalities/federationapi.go +++ b/cmd/dendrite-polylith-multi/personalities/federationapi.go @@ -29,9 +29,9 @@ func FederationAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { keyRing := fsAPI.KeyRing() federationapi.AddPublicRoutes( - base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicWellKnownAPIMux, + base.ProcessContext, base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicWellKnownAPIMux, &base.Cfg.FederationAPI, userAPI, federation, keyRing, - rsAPI, fsAPI, base.EDUServerClient(), keyAPI, + rsAPI, fsAPI, keyAPI, &base.Cfg.MSCs, nil, ) diff --git a/cmd/dendritejs-pinecone/main.go b/cmd/dendritejs-pinecone/main.go index 407081f59..211b3e131 100644 --- a/cmd/dendritejs-pinecone/main.go +++ b/cmd/dendritejs-pinecone/main.go @@ -21,18 +21,13 @@ import ( "crypto/ed25519" "encoding/hex" "fmt" - "log" - "os" "syscall/js" - "time" "github.com/gorilla/mux" "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/conn" "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" - "github.com/matrix-org/dendrite/eduserver" - "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/keyserver" @@ -48,6 +43,7 @@ import ( _ "github.com/matrix-org/go-sqlite3-js" + pineconeConnections "github.com/matrix-org/pinecone/connections" pineconeRouter "github.com/matrix-org/pinecone/router" pineconeSessions "github.com/matrix-org/pinecone/sessions" ) @@ -156,9 +152,10 @@ func startup() { sk := generateKey() pk := sk.Public().(ed25519.PublicKey) - logger := log.New(os.Stdout, "", 0) - pRouter := pineconeRouter.NewRouter(logger, sk, false) - pSessions := pineconeSessions.NewSessions(logger, pRouter) + pRouter := pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk, false) + pSessions := pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), pRouter, []string{"matrix"}) + pManager := pineconeConnections.NewConnectionManager(pRouter) + pManager.AddPeer("wss://pinecone.matrix.org/public") cfg := &config.Dendrite{} cfg.Defaults(true) @@ -193,7 +190,6 @@ func startup() { userAPI := userapi.NewInternalAPI(base, accountDB, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient()) keyAPI.SetUserAPI(userAPI) - eduInputAPI := eduserver.NewInternalAPI(base, cache.New(), userAPI) asQuery := appservice.NewInternalAPI( base, userAPI, rsAPI, ) @@ -208,12 +204,11 @@ func startup() { FedClient: federation, KeyRing: keyRing, - AppserviceAPI: asQuery, - EDUInternalAPI: eduInputAPI, - FederationAPI: fedSenderAPI, - RoomserverAPI: rsAPI, - UserAPI: userAPI, - KeyAPI: keyAPI, + AppserviceAPI: asQuery, + FederationAPI: fedSenderAPI, + RoomserverAPI: rsAPI, + UserAPI: userAPI, + KeyAPI: keyAPI, //ServerKeyAPI: serverKeyAPI, ExtPublicRoomsProvider: rooms.NewPineconeRoomProvider(pRouter, pSessions, fedSenderAPI, federation), } @@ -232,7 +227,7 @@ func startup() { httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux) httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux) - p2pRouter := pSessions.HTTP().Mux() + p2pRouter := pSessions.Protocol("matrix").HTTP().Mux() p2pRouter.Handle(httputil.PublicFederationPathPrefix, base.PublicFederationAPIMux) p2pRouter.Handle(httputil.PublicMediaPathPrefix, base.PublicMediaAPIMux) @@ -244,20 +239,4 @@ func startup() { } s.ListenAndServe("fetch") }() - - // Connect to the static peer - go func() { - for { - if pRouter.PeerCount(pineconeRouter.PeerTypeRemote) == 0 { - if err := conn.ConnectToPeer(pRouter, publicPeer); err != nil { - logrus.WithError(err).Error("Failed to connect to static peer") - } - } - select { - case <-base.ProcessContext.Context().Done(): - return - case <-time.After(time.Second * 5): - } - } - }() } diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 37cbb12dd..05e0f0ad9 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -24,8 +24,6 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/appservice" - "github.com/matrix-org/dendrite/eduserver" - "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/keyserver" @@ -203,7 +201,6 @@ func main() { } rsAPI := roomserver.NewInternalAPI(base) - eduInputAPI := eduserver.NewInternalAPI(base, cache.New(), userAPI) asQuery := appservice.NewInternalAPI( base, userAPI, rsAPI, ) @@ -222,7 +219,6 @@ func main() { KeyRing: &keyRing, AppserviceAPI: asQuery, - EDUInternalAPI: eduInputAPI, FederationSenderAPI: fedSenderAPI, RoomserverAPI: rsAPI, UserAPI: userAPI, diff --git a/cmd/federation-api-proxy/main.go b/cmd/federation-api-proxy/main.go deleted file mode 100644 index 7324de148..000000000 --- a/cmd/federation-api-proxy/main.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "flag" - "fmt" - "net/http" - "net/http/httputil" - "net/url" - "os" - "strings" - "time" - - log "github.com/sirupsen/logrus" -) - -const usage = `Usage: %s - -Create a single endpoint URL which remote matrix servers can be pointed at. - -The server-server API in Dendrite is split across multiple processes -which listen on multiple ports. You cannot point a Matrix server at -any of those ports, as there will be unimplemented functionality. -In addition, all server-server API processes start with the additional -path prefix '/api', which Matrix servers will be unaware of. - -This tool will proxy requests for all server-server URLs and forward -them to their respective process. It will also add the '/api' path -prefix to incoming requests. - -THIS TOOL IS FOR TESTING AND NOT INTENDED FOR PRODUCTION USE. - -Arguments: - -` - -var ( - federationAPIURL = flag.String("federation-api-url", "", "The base URL of the listening 'dendrite-federation-api-server' process. E.g. 'http://localhost:4200'") - mediaAPIURL = flag.String("media-api-server-url", "", "The base URL of the listening 'dendrite-media-api-server' process. E.g. 'http://localhost:7779'") - bindAddress = flag.String("bind-address", ":8448", "The listening port for the proxy.") - certFile = flag.String("tls-cert", "server.crt", "The PEM formatted X509 certificate to use for TLS") - keyFile = flag.String("tls-key", "server.key", "The PEM private key to use for TLS") -) - -func makeProxy(targetURL string) (*httputil.ReverseProxy, error) { - if !strings.HasSuffix(targetURL, "/") { - targetURL += "/" - } - // Check that we can parse the URL. - _, err := url.Parse(targetURL) - if err != nil { - return nil, err - } - return &httputil.ReverseProxy{ - Director: func(req *http.Request) { - // URL.Path() removes the % escaping from the path. - // The % encoding will be added back when the url is encoded - // when the request is forwarded. - // This means that we will lose any unessecary escaping from the URL. - // Pratically this means that any distinction between '%2F' and '/' - // in the URL will be lost by the time it reaches the target. - path := req.URL.Path - log.WithFields(log.Fields{ - "path": path, - "url": targetURL, - "method": req.Method, - }).Print("proxying request") - newURL, err := url.Parse(targetURL + path) - if err != nil { - // We already checked that we can parse the URL - // So this shouldn't ever get hit. - panic(err) - } - // Copy the query parameters from the request. - newURL.RawQuery = req.URL.RawQuery - req.URL = newURL - }, - }, nil -} - -func main() { - flag.Usage = func() { - fmt.Fprintf(os.Stderr, usage, os.Args[0]) - flag.PrintDefaults() - } - - flag.Parse() - - if *federationAPIURL == "" { - flag.Usage() - fmt.Fprintln(os.Stderr, "no --federation-api-url specified.") - os.Exit(1) - } - - if *mediaAPIURL == "" { - flag.Usage() - fmt.Fprintln(os.Stderr, "no --media-api-server-url specified.") - os.Exit(1) - } - - federationProxy, err := makeProxy(*federationAPIURL) - if err != nil { - panic(err) - } - - mediaProxy, err := makeProxy(*mediaAPIURL) - if err != nil { - panic(err) - } - - http.Handle("/_matrix/media/v1/", mediaProxy) - http.Handle("/", federationProxy) - - srv := &http.Server{ - Addr: *bindAddress, - ReadTimeout: 1 * time.Minute, // how long we wait for the client to send the entire request (after connection accept) - WriteTimeout: 5 * time.Minute, // how long the proxy has to write the full response - } - - fmt.Println("Proxying requests to:") - fmt.Println(" /_matrix/media/v1 => ", *mediaAPIURL+"/api/_matrix/media/v1") - fmt.Println(" /* => ", *federationAPIURL+"/api/*") - fmt.Println("Listening on ", *bindAddress) - panic(srv.ListenAndServeTLS(*certFile, *keyFile)) -} diff --git a/cmd/generate-config/main.go b/cmd/generate-config/main.go index ba5a87a7a..24085afaa 100644 --- a/cmd/generate-config/main.go +++ b/cmd/generate-config/main.go @@ -91,6 +91,10 @@ func main() { cfg.UserAPI.BCryptCost = bcrypt.MinCost cfg.Global.JetStream.InMemory = true cfg.ClientAPI.RegistrationSharedSecret = "complement" + cfg.Global.Presence = config.PresenceOptions{ + EnableInbound: true, + EnableOutbound: true, + } } j, err := yaml.Marshal(cfg) diff --git a/dendrite-config.yaml b/dendrite-config.yaml index 0236851c4..47f08c4fd 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -68,6 +68,13 @@ global: # to other servers and the federation API will not be exposed. disable_federation: false + # Configures the handling of presence events. + presence: + # Whether inbound presence events are allowed, e.g. receiving presence events from other servers + enable_inbound: false + # Whether outbound presence events are allowed, e.g. sending presence events to other servers + enable_outbound: false + # Server notices allows server admins to send messages to all users. server_notices: enabled: false @@ -187,12 +194,6 @@ client_api: threshold: 5 cooloff_ms: 500 -# Configuration for the EDU server. -edu_server: - internal_api: - listen: http://localhost:7778 # Only used in polylith deployments - connect: http://localhost:7778 # Only used in polylith deployments - # Configuration for the Federation API. federation_api: internal_api: @@ -206,12 +207,6 @@ federation_api: max_idle_conns: 2 conn_max_lifetime: -1 - # List of paths to X.509 certificates to be used by the external federation listeners. - # These certificates will be used to calculate the TLS fingerprints and other servers - # will expect the certificate to match these fingerprints. Certificates must be in PEM - # format. - federation_certificates: [] - # How many times we will try to resend a failed transaction to a specific server. The # backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds etc. send_max_retries: 16 diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index fe7127c76..116adfae6 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -94,4 +94,4 @@ For more general questions please use We ask that everyone who contributes to the project signs off their contributions, in accordance with the -[DCO](https://github.com/matrix-org/matrix-doc/blob/main/CONTRIBUTING.rst#sign-off). +[DCO](https://github.com/matrix-org/matrix-spec/blob/main/CONTRIBUTING.rst#sign-off). diff --git a/docs/INSTALL.md b/docs/INSTALL.md index a7b2e67f2..ca1316aca 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -34,6 +34,10 @@ If you want to run a polylith deployment, you also need: * A standalone [NATS Server](https://github.com/nats-io/nats-server) deployment with JetStream enabled +If you want to build it on Windows, you need `gcc` in the path: + +* [MinGW-w64](https://www.mingw-w64.org/) + ## Building Dendrite Start by cloning the code: @@ -45,9 +49,15 @@ cd dendrite Then build it: -```bash -./build.sh -``` +* Linux or UNIX-like systems: + ```bash + ./build.sh + ``` + +* Windows: + ```dos + build.cmd + ``` ## Install NATS Server @@ -263,14 +273,6 @@ This manages end-to-end encryption keys for users. ./bin/dendrite-polylith-multi --config=dendrite.yaml keyserver ``` -#### EDU server - -This manages processing EDUs such as typing, send-to-device events and presence. Clients do not talk to - -```bash -./bin/dendrite-polylith-multi --config=dendrite.yaml eduserver -``` - #### User server This manages user accounts, device access tokens and user account data, diff --git a/eduserver/api/input.go b/eduserver/api/input.go deleted file mode 100644 index 2aab107b2..000000000 --- a/eduserver/api/input.go +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package api provides the types that are used to communicate with the typing server. -package api - -import ( - "context" - - "github.com/matrix-org/gomatrixserverlib" -) - -// InputTypingEvent is an event for notifying the typing server about typing updates. -type InputTypingEvent struct { - // UserID of the user to update typing status. - UserID string `json:"user_id"` - // RoomID of the room the user is typing (or has stopped). - RoomID string `json:"room_id"` - // Typing is true if the user is typing, false if they have stopped. - Typing bool `json:"typing"` - // Timeout is the interval in milliseconds for which the user should be marked as typing. - TimeoutMS int64 `json:"timeout"` - // OriginServerTS when the server received the update. - OriginServerTS gomatrixserverlib.Timestamp `json:"origin_server_ts"` -} - -type InputSendToDeviceEvent struct { - UserID string `json:"user_id"` - DeviceID string `json:"device_id"` - gomatrixserverlib.SendToDeviceEvent -} - -// InputTypingEventRequest is a request to EDUServerInputAPI -type InputTypingEventRequest struct { - InputTypingEvent InputTypingEvent `json:"input_typing_event"` -} - -// InputTypingEventResponse is a response to InputTypingEvents -type InputTypingEventResponse struct{} - -// InputSendToDeviceEventRequest is a request to EDUServerInputAPI -type InputSendToDeviceEventRequest struct { - InputSendToDeviceEvent InputSendToDeviceEvent `json:"input_send_to_device_event"` -} - -// InputSendToDeviceEventResponse is a response to InputSendToDeviceEventRequest -type InputSendToDeviceEventResponse struct{} - -type InputReceiptEvent struct { - UserID string `json:"user_id"` - RoomID string `json:"room_id"` - EventID string `json:"event_id"` - Type string `json:"type"` - Timestamp gomatrixserverlib.Timestamp `json:"timestamp"` -} - -// InputReceiptEventRequest is a request to EDUServerInputAPI -type InputReceiptEventRequest struct { - InputReceiptEvent InputReceiptEvent `json:"input_receipt_event"` -} - -// InputReceiptEventResponse is a response to InputReceiptEventRequest -type InputReceiptEventResponse struct{} - -type InputCrossSigningKeyUpdateRequest struct { - CrossSigningKeyUpdate `json:"signing_keys"` -} - -type InputCrossSigningKeyUpdateResponse struct{} - -// EDUServerInputAPI is used to write events to the typing server. -type EDUServerInputAPI interface { - InputTypingEvent( - ctx context.Context, - request *InputTypingEventRequest, - response *InputTypingEventResponse, - ) error - - InputSendToDeviceEvent( - ctx context.Context, - request *InputSendToDeviceEventRequest, - response *InputSendToDeviceEventResponse, - ) error - - InputReceiptEvent( - ctx context.Context, - request *InputReceiptEventRequest, - response *InputReceiptEventResponse, - ) error -} diff --git a/eduserver/api/output.go b/eduserver/api/output.go deleted file mode 100644 index c6de4e01c..000000000 --- a/eduserver/api/output.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "time" - - "github.com/matrix-org/gomatrixserverlib" -) - -// OutputTypingEvent is an entry in typing server output kafka log. -// This contains the event with extra fields used to create 'm.typing' event -// in clientapi & federation. -type OutputTypingEvent struct { - // The Event for the typing edu event. - Event TypingEvent `json:"event"` - // ExpireTime is the interval after which the user should no longer be - // considered typing. Only available if Event.Typing is true. - ExpireTime *time.Time -} - -// OutputSendToDeviceEvent is an entry in the send-to-device output kafka log. -// This contains the full event content, along with the user ID and device ID -// to which it is destined. -type OutputSendToDeviceEvent struct { - UserID string `json:"user_id"` - DeviceID string `json:"device_id"` - gomatrixserverlib.SendToDeviceEvent -} - -// OutputReceiptEvent is an entry in the receipt output kafka log -type OutputReceiptEvent struct { - UserID string `json:"user_id"` - RoomID string `json:"room_id"` - EventID string `json:"event_id"` - Type string `json:"type"` - Timestamp gomatrixserverlib.Timestamp `json:"timestamp"` -} - -// OutputCrossSigningKeyUpdate is an entry in the signing key update output kafka log -type OutputCrossSigningKeyUpdate struct { - CrossSigningKeyUpdate `json:"signing_keys"` -} diff --git a/eduserver/api/types.go b/eduserver/api/types.go deleted file mode 100644 index a207580f9..000000000 --- a/eduserver/api/types.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2021 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import "github.com/matrix-org/gomatrixserverlib" - -const ( - MSigningKeyUpdate = "m.signing_key_update" -) - -type TypingEvent struct { - Type string `json:"type"` - RoomID string `json:"room_id"` - UserID string `json:"user_id"` - Typing bool `json:"typing"` -} - -type ReceiptEvent struct { - UserID string `json:"user_id"` - RoomID string `json:"room_id"` - EventID string `json:"event_id"` - Type string `json:"type"` - Timestamp gomatrixserverlib.Timestamp `json:"timestamp"` -} - -type FederationReceiptMRead struct { - User map[string]FederationReceiptData `json:"m.read"` -} - -type FederationReceiptData struct { - Data ReceiptTS `json:"data"` - EventIDs []string `json:"event_ids"` -} - -type ReceiptMRead struct { - User map[string]ReceiptTS `json:"m.read"` -} - -type ReceiptTS struct { - TS gomatrixserverlib.Timestamp `json:"ts"` -} - -type CrossSigningKeyUpdate struct { - MasterKey *gomatrixserverlib.CrossSigningKey `json:"master_key,omitempty"` - SelfSigningKey *gomatrixserverlib.CrossSigningKey `json:"self_signing_key,omitempty"` - UserID string `json:"user_id"` -} diff --git a/eduserver/api/wrapper.go b/eduserver/api/wrapper.go deleted file mode 100644 index 7907f4d39..000000000 --- a/eduserver/api/wrapper.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "context" - "encoding/json" - "time" - - "github.com/matrix-org/gomatrixserverlib" -) - -// SendTyping sends a typing event to EDU server -func SendTyping( - ctx context.Context, eduAPI EDUServerInputAPI, userID, roomID string, - typing bool, timeoutMS int64, -) error { - requestData := InputTypingEvent{ - UserID: userID, - RoomID: roomID, - Typing: typing, - TimeoutMS: timeoutMS, - OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()), - } - - var response InputTypingEventResponse - err := eduAPI.InputTypingEvent( - ctx, &InputTypingEventRequest{InputTypingEvent: requestData}, &response, - ) - - return err -} - -// SendToDevice sends a typing event to EDU server -func SendToDevice( - ctx context.Context, eduAPI EDUServerInputAPI, sender, userID, deviceID, eventType string, - message interface{}, -) error { - js, err := json.Marshal(message) - if err != nil { - return err - } - requestData := InputSendToDeviceEvent{ - UserID: userID, - DeviceID: deviceID, - SendToDeviceEvent: gomatrixserverlib.SendToDeviceEvent{ - Sender: sender, - Type: eventType, - Content: js, - }, - } - request := InputSendToDeviceEventRequest{ - InputSendToDeviceEvent: requestData, - } - response := InputSendToDeviceEventResponse{} - return eduAPI.InputSendToDeviceEvent(ctx, &request, &response) -} - -// SendReceipt sends a receipt event to EDU Server -func SendReceipt( - ctx context.Context, - eduAPI EDUServerInputAPI, userID, roomID, eventID, receiptType string, - timestamp gomatrixserverlib.Timestamp, -) error { - request := InputReceiptEventRequest{ - InputReceiptEvent: InputReceiptEvent{ - UserID: userID, - RoomID: roomID, - EventID: eventID, - Type: receiptType, - Timestamp: timestamp, - }, - } - response := InputReceiptEventResponse{} - return eduAPI.InputReceiptEvent(ctx, &request, &response) -} diff --git a/eduserver/eduserver.go b/eduserver/eduserver.go deleted file mode 100644 index 91208a400..000000000 --- a/eduserver/eduserver.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package eduserver - -import ( - "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/eduserver/api" - "github.com/matrix-org/dendrite/eduserver/cache" - "github.com/matrix-org/dendrite/eduserver/input" - "github.com/matrix-org/dendrite/eduserver/inthttp" - "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/jetstream" - userapi "github.com/matrix-org/dendrite/userapi/api" -) - -// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions -// on the given input API. -func AddInternalRoutes(internalMux *mux.Router, inputAPI api.EDUServerInputAPI) { - inthttp.AddRoutes(inputAPI, internalMux) -} - -// NewInternalAPI returns a concerete implementation of the internal API. Callers -// can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. -func NewInternalAPI( - base *base.BaseDendrite, - eduCache *cache.EDUCache, - userAPI userapi.UserInternalAPI, -) api.EDUServerInputAPI { - cfg := &base.Cfg.EDUServer - - js, _ := jetstream.Prepare(base.ProcessContext, &cfg.Matrix.JetStream) - - return &input.EDUServerInputAPI{ - Cache: eduCache, - UserAPI: userAPI, - JetStream: js, - OutputTypingEventTopic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent), - OutputSendToDeviceEventTopic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), - OutputReceiptEventTopic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent), - ServerName: cfg.Matrix.ServerName, - } -} diff --git a/eduserver/input/input.go b/eduserver/input/input.go deleted file mode 100644 index e58f0dd34..000000000 --- a/eduserver/input/input.go +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package input - -import ( - "context" - "encoding/json" - "time" - - "github.com/matrix-org/dendrite/eduserver/api" - "github.com/matrix-org/dendrite/eduserver/cache" - userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" - "github.com/nats-io/nats.go" - "github.com/sirupsen/logrus" -) - -// EDUServerInputAPI implements api.EDUServerInputAPI -type EDUServerInputAPI struct { - // Cache to store the current typing members in each room. - Cache *cache.EDUCache - // The kafka topic to output new typing events to. - OutputTypingEventTopic string - // The kafka topic to output new send to device events to. - OutputSendToDeviceEventTopic string - // The kafka topic to output new receipt events to - OutputReceiptEventTopic string - // kafka producer - JetStream nats.JetStreamContext - // Internal user query API - UserAPI userapi.UserInternalAPI - // our server name - ServerName gomatrixserverlib.ServerName -} - -// InputTypingEvent implements api.EDUServerInputAPI -func (t *EDUServerInputAPI) InputTypingEvent( - ctx context.Context, - request *api.InputTypingEventRequest, - response *api.InputTypingEventResponse, -) error { - ite := &request.InputTypingEvent - if ite.Typing { - // user is typing, update our current state of users typing. - expireTime := ite.OriginServerTS.Time().Add( - time.Duration(ite.TimeoutMS) * time.Millisecond, - ) - t.Cache.AddTypingUser(ite.UserID, ite.RoomID, &expireTime) - } else { - t.Cache.RemoveUser(ite.UserID, ite.RoomID) - } - - return t.sendTypingEvent(ite) -} - -// InputTypingEvent implements api.EDUServerInputAPI -func (t *EDUServerInputAPI) InputSendToDeviceEvent( - ctx context.Context, - request *api.InputSendToDeviceEventRequest, - response *api.InputSendToDeviceEventResponse, -) error { - ise := &request.InputSendToDeviceEvent - return t.sendToDeviceEvent(ise) -} - -func (t *EDUServerInputAPI) sendTypingEvent(ite *api.InputTypingEvent) error { - ev := &api.TypingEvent{ - Type: gomatrixserverlib.MTyping, - RoomID: ite.RoomID, - UserID: ite.UserID, - Typing: ite.Typing, - } - ote := &api.OutputTypingEvent{ - Event: *ev, - } - - if ev.Typing { - expireTime := ite.OriginServerTS.Time().Add( - time.Duration(ite.TimeoutMS) * time.Millisecond, - ) - ote.ExpireTime = &expireTime - } - - eventJSON, err := json.Marshal(ote) - if err != nil { - return err - } - logrus.WithFields(logrus.Fields{ - "room_id": ite.RoomID, - "user_id": ite.UserID, - "typing": ite.Typing, - }).Tracef("Producing to topic '%s'", t.OutputTypingEventTopic) - - _, err = t.JetStream.PublishMsg(&nats.Msg{ - Subject: t.OutputTypingEventTopic, - Header: nats.Header{}, - Data: eventJSON, - }) - return err -} - -func (t *EDUServerInputAPI) sendToDeviceEvent(ise *api.InputSendToDeviceEvent) error { - devices := []string{} - _, domain, err := gomatrixserverlib.SplitID('@', ise.UserID) - if err != nil { - return err - } - - // If the event is targeted locally then we want to expand the wildcard - // out into individual device IDs so that we can send them to each respective - // device. If the event isn't targeted locally then we can't expand the - // wildcard as we don't know about the remote devices, so instead we leave it - // as-is, so that the federation sender can send it on with the wildcard intact. - if domain == t.ServerName && ise.DeviceID == "*" { - var res userapi.QueryDevicesResponse - err = t.UserAPI.QueryDevices(context.TODO(), &userapi.QueryDevicesRequest{ - UserID: ise.UserID, - }, &res) - if err != nil { - return err - } - for _, dev := range res.Devices { - devices = append(devices, dev.ID) - } - } else { - devices = append(devices, ise.DeviceID) - } - - logrus.WithFields(logrus.Fields{ - "user_id": ise.UserID, - "num_devices": len(devices), - "type": ise.Type, - }).Tracef("Producing to topic '%s'", t.OutputSendToDeviceEventTopic) - for _, device := range devices { - ote := &api.OutputSendToDeviceEvent{ - UserID: ise.UserID, - DeviceID: device, - SendToDeviceEvent: ise.SendToDeviceEvent, - } - - eventJSON, err := json.Marshal(ote) - if err != nil { - logrus.WithError(err).Error("sendToDevice failed json.Marshal") - return err - } - - if _, err = t.JetStream.PublishMsg(&nats.Msg{ - Subject: t.OutputSendToDeviceEventTopic, - Data: eventJSON, - }); err != nil { - logrus.WithError(err).Error("sendToDevice failed t.Producer.SendMessage") - return err - } - } - - return nil -} - -// InputReceiptEvent implements api.EDUServerInputAPI -// TODO: Intelligently batch requests sent by the same user (e.g wait a few milliseconds before emitting output events) -func (t *EDUServerInputAPI) InputReceiptEvent( - ctx context.Context, - request *api.InputReceiptEventRequest, - response *api.InputReceiptEventResponse, -) error { - logrus.WithFields(logrus.Fields{}).Tracef("Producing to topic '%s'", t.OutputReceiptEventTopic) - output := &api.OutputReceiptEvent{ - UserID: request.InputReceiptEvent.UserID, - RoomID: request.InputReceiptEvent.RoomID, - EventID: request.InputReceiptEvent.EventID, - Type: request.InputReceiptEvent.Type, - Timestamp: request.InputReceiptEvent.Timestamp, - } - js, err := json.Marshal(output) - if err != nil { - return err - } - - _, err = t.JetStream.PublishMsg(&nats.Msg{ - Subject: t.OutputReceiptEventTopic, - Data: js, - }) - return err -} diff --git a/eduserver/inthttp/client.go b/eduserver/inthttp/client.go deleted file mode 100644 index 0690ed827..000000000 --- a/eduserver/inthttp/client.go +++ /dev/null @@ -1,70 +0,0 @@ -package inthttp - -import ( - "context" - "errors" - "net/http" - - "github.com/matrix-org/dendrite/eduserver/api" - "github.com/matrix-org/dendrite/internal/httputil" - "github.com/opentracing/opentracing-go" -) - -// HTTP paths for the internal HTTP APIs -const ( - EDUServerInputTypingEventPath = "/eduserver/input" - EDUServerInputSendToDeviceEventPath = "/eduserver/sendToDevice" - EDUServerInputReceiptEventPath = "/eduserver/receipt" -) - -// NewEDUServerClient creates a EDUServerInputAPI implemented by talking to a HTTP POST API. -func NewEDUServerClient(eduServerURL string, httpClient *http.Client) (api.EDUServerInputAPI, error) { - if httpClient == nil { - return nil, errors.New("NewEDUServerClient: httpClient is ") - } - return &httpEDUServerInputAPI{eduServerURL, httpClient}, nil -} - -type httpEDUServerInputAPI struct { - eduServerURL string - httpClient *http.Client -} - -// InputTypingEvent implements EDUServerInputAPI -func (h *httpEDUServerInputAPI) InputTypingEvent( - ctx context.Context, - request *api.InputTypingEventRequest, - response *api.InputTypingEventResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "InputTypingEvent") - defer span.Finish() - - apiURL := h.eduServerURL + EDUServerInputTypingEventPath - return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// InputSendToDeviceEvent implements EDUServerInputAPI -func (h *httpEDUServerInputAPI) InputSendToDeviceEvent( - ctx context.Context, - request *api.InputSendToDeviceEventRequest, - response *api.InputSendToDeviceEventResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "InputSendToDeviceEvent") - defer span.Finish() - - apiURL := h.eduServerURL + EDUServerInputSendToDeviceEventPath - return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// InputSendToDeviceEvent implements EDUServerInputAPI -func (h *httpEDUServerInputAPI) InputReceiptEvent( - ctx context.Context, - request *api.InputReceiptEventRequest, - response *api.InputReceiptEventResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "InputReceiptEventPath") - defer span.Finish() - - apiURL := h.eduServerURL + EDUServerInputReceiptEventPath - return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} diff --git a/eduserver/inthttp/server.go b/eduserver/inthttp/server.go deleted file mode 100644 index a34943750..000000000 --- a/eduserver/inthttp/server.go +++ /dev/null @@ -1,54 +0,0 @@ -package inthttp - -import ( - "encoding/json" - "net/http" - - "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/eduserver/api" - "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/util" -) - -// AddRoutes adds the EDUServerInputAPI handlers to the http.ServeMux. -func AddRoutes(t api.EDUServerInputAPI, internalAPIMux *mux.Router) { - internalAPIMux.Handle(EDUServerInputTypingEventPath, - httputil.MakeInternalAPI("inputTypingEvents", func(req *http.Request) util.JSONResponse { - var request api.InputTypingEventRequest - var response api.InputTypingEventResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.MessageResponse(http.StatusBadRequest, err.Error()) - } - if err := t.InputTypingEvent(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle(EDUServerInputSendToDeviceEventPath, - httputil.MakeInternalAPI("inputSendToDeviceEvents", func(req *http.Request) util.JSONResponse { - var request api.InputSendToDeviceEventRequest - var response api.InputSendToDeviceEventResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.MessageResponse(http.StatusBadRequest, err.Error()) - } - if err := t.InputSendToDeviceEvent(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle(EDUServerInputReceiptEventPath, - httputil.MakeInternalAPI("inputReceiptEvent", func(req *http.Request) util.JSONResponse { - var request api.InputReceiptEventRequest - var response api.InputReceiptEventResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.MessageResponse(http.StatusBadRequest, err.Error()) - } - if err := t.InputReceiptEvent(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) -} diff --git a/federationapi/consumers/eduserver.go b/federationapi/consumers/eduserver.go deleted file mode 100644 index e14e60f47..000000000 --- a/federationapi/consumers/eduserver.go +++ /dev/null @@ -1,257 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package consumers - -import ( - "context" - "encoding/json" - - "github.com/matrix-org/dendrite/eduserver/api" - "github.com/matrix-org/dendrite/federationapi/queue" - "github.com/matrix-org/dendrite/federationapi/storage" - "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/setup/jetstream" - "github.com/matrix-org/dendrite/setup/process" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" - "github.com/nats-io/nats.go" - log "github.com/sirupsen/logrus" -) - -// OutputEDUConsumer consumes events that originate in EDU server. -type OutputEDUConsumer struct { - ctx context.Context - jetstream nats.JetStreamContext - durable string - db storage.Database - queues *queue.OutgoingQueues - ServerName gomatrixserverlib.ServerName - typingTopic string - sendToDeviceTopic string - receiptTopic string -} - -// NewOutputEDUConsumer creates a new OutputEDUConsumer. Call Start() to begin consuming from EDU servers. -func NewOutputEDUConsumer( - process *process.ProcessContext, - cfg *config.FederationAPI, - js nats.JetStreamContext, - queues *queue.OutgoingQueues, - store storage.Database, -) *OutputEDUConsumer { - return &OutputEDUConsumer{ - ctx: process.Context(), - jetstream: js, - queues: queues, - db: store, - ServerName: cfg.Matrix.ServerName, - durable: cfg.Matrix.JetStream.Durable("FederationAPIEDUServerConsumer"), - typingTopic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent), - sendToDeviceTopic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), - receiptTopic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent), - } -} - -// Start consuming from EDU servers -func (t *OutputEDUConsumer) Start() error { - if err := jetstream.JetStreamConsumer( - t.ctx, t.jetstream, t.typingTopic, t.durable, t.onTypingEvent, - nats.DeliverAll(), nats.ManualAck(), - ); err != nil { - return err - } - if err := jetstream.JetStreamConsumer( - t.ctx, t.jetstream, t.sendToDeviceTopic, t.durable, t.onSendToDeviceEvent, - nats.DeliverAll(), nats.ManualAck(), - ); err != nil { - return err - } - if err := jetstream.JetStreamConsumer( - t.ctx, t.jetstream, t.receiptTopic, t.durable, t.onReceiptEvent, - nats.DeliverAll(), nats.ManualAck(), - ); err != nil { - return err - } - return nil -} - -// onSendToDeviceEvent is called in response to a message received on the -// send-to-device events topic from the EDU server. -func (t *OutputEDUConsumer) onSendToDeviceEvent(ctx context.Context, msg *nats.Msg) bool { - // Extract the send-to-device event from msg. - var ote api.OutputSendToDeviceEvent - if err := json.Unmarshal(msg.Data, &ote); err != nil { - log.WithError(err).Errorf("eduserver output log: message parse failed (expected send-to-device)") - return true - } - - // only send send-to-device events which originated from us - _, originServerName, err := gomatrixserverlib.SplitID('@', ote.Sender) - if err != nil { - log.WithError(err).WithField("user_id", ote.Sender).Error("Failed to extract domain from send-to-device sender") - return true - } - if originServerName != t.ServerName { - log.WithField("other_server", originServerName).Info("Suppressing send-to-device: originated elsewhere") - return true - } - - _, destServerName, err := gomatrixserverlib.SplitID('@', ote.UserID) - if err != nil { - log.WithError(err).WithField("user_id", ote.UserID).Error("Failed to extract domain from send-to-device destination") - return true - } - - // Pack the EDU and marshal it - edu := &gomatrixserverlib.EDU{ - Type: gomatrixserverlib.MDirectToDevice, - Origin: string(t.ServerName), - } - tdm := gomatrixserverlib.ToDeviceMessage{ - Sender: ote.Sender, - Type: ote.Type, - MessageID: util.RandomString(32), - Messages: map[string]map[string]json.RawMessage{ - ote.UserID: { - ote.DeviceID: ote.Content, - }, - }, - } - if edu.Content, err = json.Marshal(tdm); err != nil { - log.WithError(err).Error("failed to marshal EDU JSON") - return true - } - - log.Debugf("Sending send-to-device message into %q destination queue", destServerName) - if err := t.queues.SendEDU(edu, t.ServerName, []gomatrixserverlib.ServerName{destServerName}); err != nil { - log.WithError(err).Error("failed to send EDU") - return false - } - - return true -} - -// onTypingEvent is called in response to a message received on the typing -// events topic from the EDU server. -func (t *OutputEDUConsumer) onTypingEvent(ctx context.Context, msg *nats.Msg) bool { - // Extract the typing event from msg. - var ote api.OutputTypingEvent - if err := json.Unmarshal(msg.Data, &ote); err != nil { - // Skip this msg but continue processing messages. - log.WithError(err).Errorf("eduserver output log: message parse failed (expected typing)") - _ = msg.Ack() - return true - } - - // only send typing events which originated from us - _, typingServerName, err := gomatrixserverlib.SplitID('@', ote.Event.UserID) - if err != nil { - log.WithError(err).WithField("user_id", ote.Event.UserID).Error("Failed to extract domain from typing sender") - _ = msg.Ack() - return true - } - if typingServerName != t.ServerName { - return true - } - - joined, err := t.db.GetJoinedHosts(ctx, ote.Event.RoomID) - if err != nil { - log.WithError(err).WithField("room_id", ote.Event.RoomID).Error("failed to get joined hosts for room") - return false - } - - names := make([]gomatrixserverlib.ServerName, len(joined)) - for i := range joined { - names[i] = joined[i].ServerName - } - - edu := &gomatrixserverlib.EDU{Type: ote.Event.Type} - if edu.Content, err = json.Marshal(map[string]interface{}{ - "room_id": ote.Event.RoomID, - "user_id": ote.Event.UserID, - "typing": ote.Event.Typing, - }); err != nil { - log.WithError(err).Error("failed to marshal EDU JSON") - return true - } - - if err := t.queues.SendEDU(edu, t.ServerName, names); err != nil { - log.WithError(err).Error("failed to send EDU") - return false - } - - return true -} - -// onReceiptEvent is called in response to a message received on the receipt -// events topic from the EDU server. -func (t *OutputEDUConsumer) onReceiptEvent(ctx context.Context, msg *nats.Msg) bool { - // Extract the typing event from msg. - var receipt api.OutputReceiptEvent - if err := json.Unmarshal(msg.Data, &receipt); err != nil { - // Skip this msg but continue processing messages. - log.WithError(err).Errorf("eduserver output log: message parse failed (expected receipt)") - return true - } - - // only send receipt events which originated from us - _, receiptServerName, err := gomatrixserverlib.SplitID('@', receipt.UserID) - if err != nil { - log.WithError(err).WithField("user_id", receipt.UserID).Error("failed to extract domain from receipt sender") - return true - } - if receiptServerName != t.ServerName { - return true - } - - joined, err := t.db.GetJoinedHosts(ctx, receipt.RoomID) - if err != nil { - log.WithError(err).WithField("room_id", receipt.RoomID).Error("failed to get joined hosts for room") - return false - } - - names := make([]gomatrixserverlib.ServerName, len(joined)) - for i := range joined { - names[i] = joined[i].ServerName - } - - content := map[string]api.FederationReceiptMRead{} - content[receipt.RoomID] = api.FederationReceiptMRead{ - User: map[string]api.FederationReceiptData{ - receipt.UserID: { - Data: api.ReceiptTS{ - TS: receipt.Timestamp, - }, - EventIDs: []string{receipt.EventID}, - }, - }, - } - - edu := &gomatrixserverlib.EDU{ - Type: gomatrixserverlib.MReceipt, - Origin: string(t.ServerName), - } - if edu.Content, err = json.Marshal(content); err != nil { - log.WithError(err).Error("failed to marshal EDU JSON") - return true - } - - if err := t.queues.SendEDU(edu, t.ServerName, names); err != nil { - log.WithError(err).Error("failed to send EDU") - return false - } - - return true -} diff --git a/federationapi/consumers/keychange.go b/federationapi/consumers/keychange.go index 94e454359..0ece18e97 100644 --- a/federationapi/consumers/keychange.go +++ b/federationapi/consumers/keychange.go @@ -18,9 +18,9 @@ import ( "context" "encoding/json" - eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/federationapi/queue" "github.com/matrix-org/dendrite/federationapi/storage" + "github.com/matrix-org/dendrite/federationapi/types" "github.com/matrix-org/dendrite/keyserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" @@ -190,7 +190,7 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool { // Pack the EDU and marshal it edu := &gomatrixserverlib.EDU{ - Type: eduserverAPI.MSigningKeyUpdate, + Type: types.MSigningKeyUpdate, Origin: string(t.serverName), } if edu.Content, err = json.Marshal(output); err != nil { diff --git a/federationapi/consumers/presence.go b/federationapi/consumers/presence.go new file mode 100644 index 000000000..bfce1b28b --- /dev/null +++ b/federationapi/consumers/presence.go @@ -0,0 +1,143 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package consumers + +import ( + "context" + "encoding/json" + "strconv" + + "github.com/matrix-org/dendrite/federationapi/queue" + "github.com/matrix-org/dendrite/federationapi/storage" + fedTypes "github.com/matrix-org/dendrite/federationapi/types" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/gomatrixserverlib" + "github.com/nats-io/nats.go" + log "github.com/sirupsen/logrus" +) + +// OutputReceiptConsumer consumes events that originate in the clientapi. +type OutputPresenceConsumer struct { + ctx context.Context + jetstream nats.JetStreamContext + durable string + db storage.Database + queues *queue.OutgoingQueues + ServerName gomatrixserverlib.ServerName + topic string + outboundPresenceEnabled bool +} + +// NewOutputPresenceConsumer creates a new OutputPresenceConsumer. Call Start() to begin consuming events. +func NewOutputPresenceConsumer( + process *process.ProcessContext, + cfg *config.FederationAPI, + js nats.JetStreamContext, + queues *queue.OutgoingQueues, + store storage.Database, +) *OutputPresenceConsumer { + return &OutputPresenceConsumer{ + ctx: process.Context(), + jetstream: js, + queues: queues, + db: store, + ServerName: cfg.Matrix.ServerName, + durable: cfg.Matrix.JetStream.Durable("FederationAPIPresenceConsumer"), + topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent), + outboundPresenceEnabled: cfg.Matrix.Presence.EnableOutbound, + } +} + +// Start consuming from the clientapi +func (t *OutputPresenceConsumer) Start() error { + if !t.outboundPresenceEnabled { + return nil + } + return jetstream.JetStreamConsumer( + t.ctx, t.jetstream, t.topic, t.durable, t.onMessage, + nats.DeliverAll(), nats.ManualAck(), nats.HeadersOnly(), + ) +} + +// onMessage is called in response to a message received on the presence +// events topic from the client api. +func (t *OutputPresenceConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool { + // only send presence events which originated from us + userID := msg.Header.Get(jetstream.UserID) + _, serverName, err := gomatrixserverlib.SplitID('@', userID) + if err != nil { + log.WithError(err).WithField("user_id", userID).Error("failed to extract domain from receipt sender") + return true + } + if serverName != t.ServerName { + return true + } + + presence := msg.Header.Get("presence") + + ts, err := strconv.Atoi(msg.Header.Get("last_active_ts")) + if err != nil { + return true + } + + joined, err := t.db.GetAllJoinedHosts(ctx) + if err != nil { + log.WithError(err).Error("failed to get joined hosts") + return true + } + if len(joined) == 0 { + return true + } + + var statusMsg *string = nil + if data, ok := msg.Header["status_msg"]; ok && len(data) > 0 { + status := msg.Header.Get("status_msg") + statusMsg = &status + } + + p := types.PresenceInternal{LastActiveTS: gomatrixserverlib.Timestamp(ts)} + + content := fedTypes.Presence{ + Push: []fedTypes.PresenceContent{ + { + CurrentlyActive: p.CurrentlyActive(), + LastActiveAgo: p.LastActiveAgo(), + Presence: presence, + StatusMsg: statusMsg, + UserID: userID, + }, + }, + } + + edu := &gomatrixserverlib.EDU{ + Type: gomatrixserverlib.MPresence, + Origin: string(t.ServerName), + } + if edu.Content, err = json.Marshal(content); err != nil { + log.WithError(err).Error("failed to marshal EDU JSON") + return true + } + + log.Debugf("sending presence EDU to %d servers", len(joined)) + if err = t.queues.SendEDU(edu, t.ServerName, joined); err != nil { + log.WithError(err).Error("failed to send EDU") + return false + } + + return true +} diff --git a/federationapi/consumers/receipts.go b/federationapi/consumers/receipts.go new file mode 100644 index 000000000..9300451eb --- /dev/null +++ b/federationapi/consumers/receipts.go @@ -0,0 +1,141 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package consumers + +import ( + "context" + "encoding/json" + "strconv" + + "github.com/getsentry/sentry-go" + "github.com/matrix-org/dendrite/federationapi/queue" + "github.com/matrix-org/dendrite/federationapi/storage" + fedTypes "github.com/matrix-org/dendrite/federationapi/types" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" + syncTypes "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/gomatrixserverlib" + "github.com/nats-io/nats.go" + log "github.com/sirupsen/logrus" +) + +// OutputReceiptConsumer consumes events that originate in the clientapi. +type OutputReceiptConsumer struct { + ctx context.Context + jetstream nats.JetStreamContext + durable string + db storage.Database + queues *queue.OutgoingQueues + ServerName gomatrixserverlib.ServerName + topic string +} + +// NewOutputReceiptConsumer creates a new OutputReceiptConsumer. Call Start() to begin consuming typing events. +func NewOutputReceiptConsumer( + process *process.ProcessContext, + cfg *config.FederationAPI, + js nats.JetStreamContext, + queues *queue.OutgoingQueues, + store storage.Database, +) *OutputReceiptConsumer { + return &OutputReceiptConsumer{ + ctx: process.Context(), + jetstream: js, + queues: queues, + db: store, + ServerName: cfg.Matrix.ServerName, + durable: cfg.Matrix.JetStream.Durable("FederationAPIReceiptConsumer"), + topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent), + } +} + +// Start consuming from the clientapi +func (t *OutputReceiptConsumer) Start() error { + return jetstream.JetStreamConsumer( + t.ctx, t.jetstream, t.topic, t.durable, t.onMessage, + nats.DeliverAll(), nats.ManualAck(), nats.HeadersOnly(), + ) +} + +// onMessage is called in response to a message received on the receipt +// events topic from the client api. +func (t *OutputReceiptConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool { + receipt := syncTypes.OutputReceiptEvent{ + UserID: msg.Header.Get(jetstream.UserID), + RoomID: msg.Header.Get(jetstream.RoomID), + EventID: msg.Header.Get(jetstream.EventID), + Type: msg.Header.Get("type"), + } + + // only send receipt events which originated from us + _, receiptServerName, err := gomatrixserverlib.SplitID('@', receipt.UserID) + if err != nil { + log.WithError(err).WithField("user_id", receipt.UserID).Error("failed to extract domain from receipt sender") + return true + } + if receiptServerName != t.ServerName { + return true + } + + timestamp, err := strconv.Atoi(msg.Header.Get("timestamp")) + if err != nil { + // If the message was invalid, log it and move on to the next message in the stream + log.WithError(err).Errorf("EDU output log: message parse failure") + sentry.CaptureException(err) + return true + } + + receipt.Timestamp = gomatrixserverlib.Timestamp(timestamp) + + joined, err := t.db.GetJoinedHosts(ctx, receipt.RoomID) + if err != nil { + log.WithError(err).WithField("room_id", receipt.RoomID).Error("failed to get joined hosts for room") + return false + } + + names := make([]gomatrixserverlib.ServerName, len(joined)) + for i := range joined { + names[i] = joined[i].ServerName + } + + content := map[string]fedTypes.FederationReceiptMRead{} + content[receipt.RoomID] = fedTypes.FederationReceiptMRead{ + User: map[string]fedTypes.FederationReceiptData{ + receipt.UserID: { + Data: fedTypes.ReceiptTS{ + TS: receipt.Timestamp, + }, + EventIDs: []string{receipt.EventID}, + }, + }, + } + + edu := &gomatrixserverlib.EDU{ + Type: gomatrixserverlib.MReceipt, + Origin: string(t.ServerName), + } + if edu.Content, err = json.Marshal(content); err != nil { + log.WithError(err).Error("failed to marshal EDU JSON") + return true + } + + if err := t.queues.SendEDU(edu, t.ServerName, names); err != nil { + log.WithError(err).Error("failed to send EDU") + return false + } + + return true +} diff --git a/federationapi/consumers/sendtodevice.go b/federationapi/consumers/sendtodevice.go new file mode 100644 index 000000000..84c9f620d --- /dev/null +++ b/federationapi/consumers/sendtodevice.go @@ -0,0 +1,125 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package consumers + +import ( + "context" + "encoding/json" + + "github.com/matrix-org/dendrite/federationapi/queue" + "github.com/matrix-org/dendrite/federationapi/storage" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" + syncTypes "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/nats-io/nats.go" + log "github.com/sirupsen/logrus" +) + +// OutputSendToDeviceConsumer consumes events that originate in the clientapi. +type OutputSendToDeviceConsumer struct { + ctx context.Context + jetstream nats.JetStreamContext + durable string + db storage.Database + queues *queue.OutgoingQueues + ServerName gomatrixserverlib.ServerName + topic string +} + +// NewOutputSendToDeviceConsumer creates a new OutputSendToDeviceConsumer. Call Start() to begin consuming send-to-device events. +func NewOutputSendToDeviceConsumer( + process *process.ProcessContext, + cfg *config.FederationAPI, + js nats.JetStreamContext, + queues *queue.OutgoingQueues, + store storage.Database, +) *OutputSendToDeviceConsumer { + return &OutputSendToDeviceConsumer{ + ctx: process.Context(), + jetstream: js, + queues: queues, + db: store, + ServerName: cfg.Matrix.ServerName, + durable: cfg.Matrix.JetStream.Durable("FederationAPIESendToDeviceConsumer"), + topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), + } +} + +// Start consuming from the client api +func (t *OutputSendToDeviceConsumer) Start() error { + return jetstream.JetStreamConsumer( + t.ctx, t.jetstream, t.topic, t.durable, t.onMessage, + nats.DeliverAll(), nats.ManualAck(), + ) +} + +// onMessage is called in response to a message received on the +// send-to-device events topic from the client api. +func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool { + // only send send-to-device events which originated from us + sender := msg.Header.Get("sender") + _, originServerName, err := gomatrixserverlib.SplitID('@', sender) + if err != nil { + log.WithError(err).WithField("user_id", sender).Error("Failed to extract domain from send-to-device sender") + return true + } + if originServerName != t.ServerName { + log.WithField("other_server", originServerName).Info("Suppressing send-to-device: originated elsewhere") + return true + } + // Extract the send-to-device event from msg. + var ote syncTypes.OutputSendToDeviceEvent + if err = json.Unmarshal(msg.Data, &ote); err != nil { + log.WithError(err).Errorf("output log: message parse failed (expected send-to-device)") + return true + } + + _, destServerName, err := gomatrixserverlib.SplitID('@', ote.UserID) + if err != nil { + log.WithError(err).WithField("user_id", ote.UserID).Error("Failed to extract domain from send-to-device destination") + return true + } + + // Pack the EDU and marshal it + edu := &gomatrixserverlib.EDU{ + Type: gomatrixserverlib.MDirectToDevice, + Origin: string(t.ServerName), + } + tdm := gomatrixserverlib.ToDeviceMessage{ + Sender: ote.Sender, + Type: ote.Type, + MessageID: util.RandomString(32), + Messages: map[string]map[string]json.RawMessage{ + ote.UserID: { + ote.DeviceID: ote.Content, + }, + }, + } + if edu.Content, err = json.Marshal(tdm); err != nil { + log.WithError(err).Error("failed to marshal EDU JSON") + return true + } + + log.Debugf("Sending send-to-device message into %q destination queue", destServerName) + if err := t.queues.SendEDU(edu, t.ServerName, []gomatrixserverlib.ServerName{destServerName}); err != nil { + log.WithError(err).Error("failed to send EDU") + return false + } + + return true +} diff --git a/federationapi/consumers/typing.go b/federationapi/consumers/typing.go new file mode 100644 index 000000000..428e1a867 --- /dev/null +++ b/federationapi/consumers/typing.go @@ -0,0 +1,119 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package consumers + +import ( + "context" + "encoding/json" + "strconv" + + "github.com/matrix-org/dendrite/federationapi/queue" + "github.com/matrix-org/dendrite/federationapi/storage" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" + "github.com/matrix-org/gomatrixserverlib" + "github.com/nats-io/nats.go" + log "github.com/sirupsen/logrus" +) + +// OutputTypingConsumer consumes events that originate in the clientapi. +type OutputTypingConsumer struct { + ctx context.Context + jetstream nats.JetStreamContext + durable string + db storage.Database + queues *queue.OutgoingQueues + ServerName gomatrixserverlib.ServerName + topic string +} + +// NewOutputTypingConsumer creates a new OutputTypingConsumer. Call Start() to begin consuming typing events. +func NewOutputTypingConsumer( + process *process.ProcessContext, + cfg *config.FederationAPI, + js nats.JetStreamContext, + queues *queue.OutgoingQueues, + store storage.Database, +) *OutputTypingConsumer { + return &OutputTypingConsumer{ + ctx: process.Context(), + jetstream: js, + queues: queues, + db: store, + ServerName: cfg.Matrix.ServerName, + durable: cfg.Matrix.JetStream.Durable("FederationAPITypingConsumer"), + topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent), + } +} + +// Start consuming from the clientapi +func (t *OutputTypingConsumer) Start() error { + return jetstream.JetStreamConsumer( + t.ctx, t.jetstream, t.topic, t.durable, t.onMessage, + nats.DeliverAll(), nats.ManualAck(), nats.HeadersOnly(), + ) +} + +// onMessage is called in response to a message received on the typing +// events topic from the client api. +func (t *OutputTypingConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool { + // Extract the typing event from msg. + roomID := msg.Header.Get(jetstream.RoomID) + userID := msg.Header.Get(jetstream.UserID) + typing, err := strconv.ParseBool(msg.Header.Get("typing")) + if err != nil { + log.WithError(err).Errorf("EDU output log: typing parse failure") + return true + } + + // only send typing events which originated from us + _, typingServerName, err := gomatrixserverlib.SplitID('@', userID) + if err != nil { + log.WithError(err).WithField("user_id", userID).Error("Failed to extract domain from typing sender") + _ = msg.Ack() + return true + } + if typingServerName != t.ServerName { + return true + } + + joined, err := t.db.GetJoinedHosts(ctx, roomID) + if err != nil { + log.WithError(err).WithField("room_id", roomID).Error("failed to get joined hosts for room") + return false + } + + names := make([]gomatrixserverlib.ServerName, len(joined)) + for i := range joined { + names[i] = joined[i].ServerName + } + + edu := &gomatrixserverlib.EDU{Type: "m.typing"} + if edu.Content, err = json.Marshal(map[string]interface{}{ + "room_id": roomID, + "user_id": userID, + "typing": typing, + }); err != nil { + log.WithError(err).Error("failed to marshal EDU JSON") + return true + } + if err := t.queues.SendEDU(edu, t.ServerName, names); err != nil { + log.WithError(err).Error("failed to send EDU") + return false + } + + return true +} diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index b7f93ecb9..5bfe237a8 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -16,12 +16,12 @@ package federationapi import ( "github.com/gorilla/mux" - eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/federationapi/api" federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/federationapi/consumers" "github.com/matrix-org/dendrite/federationapi/internal" "github.com/matrix-org/dendrite/federationapi/inthttp" + "github.com/matrix-org/dendrite/federationapi/producers" "github.com/matrix-org/dendrite/federationapi/queue" "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/federationapi/storage" @@ -31,6 +31,7 @@ import ( "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/sirupsen/logrus" @@ -46,6 +47,7 @@ func AddInternalRoutes(router *mux.Router, intAPI api.FederationInternalAPI) { // AddPublicRoutes sets up and registers HTTP handlers on the base API muxes for the FederationAPI component. func AddPublicRoutes( + process *process.ProcessContext, fedRouter, keyRouter, wellKnownRouter *mux.Router, cfg *config.FederationAPI, userAPI userapi.UserInternalAPI, @@ -53,16 +55,27 @@ func AddPublicRoutes( keyRing gomatrixserverlib.JSONVerifier, rsAPI roomserverAPI.RoomserverInternalAPI, federationAPI federationAPI.FederationInternalAPI, - eduAPI eduserverAPI.EDUServerInputAPI, keyAPI keyserverAPI.KeyInternalAPI, mscCfg *config.MSCs, servers federationAPI.ServersInRoomProvider, ) { + + js, _ := jetstream.Prepare(process, &cfg.Matrix.JetStream) + producer := &producers.SyncAPIProducer{ + JetStream: js, + TopicReceiptEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent), + TopicSendToDeviceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), + TopicTypingEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent), + TopicPresenceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent), + ServerName: cfg.Matrix.ServerName, + UserAPI: userAPI, + } + routing.Setup( fedRouter, keyRouter, wellKnownRouter, cfg, rsAPI, - eduAPI, federationAPI, keyRing, + federationAPI, keyRing, federation, userAPI, keyAPI, mscCfg, - servers, + servers, producer, ) } @@ -112,19 +125,36 @@ func NewInternalAPI( if err = rsConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start room server consumer") } - - tsConsumer := consumers.NewOutputEDUConsumer( + tsConsumer := consumers.NewOutputSendToDeviceConsumer( base.ProcessContext, cfg, js, queues, federationDB, ) - if err := tsConsumer.Start(); err != nil { - logrus.WithError(err).Panic("failed to start typing server consumer") + if err = tsConsumer.Start(); err != nil { + logrus.WithError(err).Panic("failed to start send-to-device consumer") + } + receiptConsumer := consumers.NewOutputReceiptConsumer( + base.ProcessContext, cfg, js, queues, federationDB, + ) + if err = receiptConsumer.Start(); err != nil { + logrus.WithError(err).Panic("failed to start receipt consumer") + } + typingConsumer := consumers.NewOutputTypingConsumer( + base.ProcessContext, cfg, js, queues, federationDB, + ) + if err = typingConsumer.Start(); err != nil { + logrus.WithError(err).Panic("failed to start typing consumer") } keyConsumer := consumers.NewKeyChangeConsumer( base.ProcessContext, &base.Cfg.KeyServer, js, queues, federationDB, rsAPI, ) - if err := keyConsumer.Start(); err != nil { + if err = keyConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start key server consumer") } + presenceConsumer := consumers.NewOutputPresenceConsumer( + base.ProcessContext, cfg, js, queues, federationDB, + ) + if err = presenceConsumer.Start(); err != nil { + logrus.WithError(err).Panic("failed to start presence consumer") + } return internal.NewFederationInternalAPI(federationDB, cfg, rsAPI, federation, stats, caches, queues, keyRing) } diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go index c660f12e0..833359c11 100644 --- a/federationapi/federationapi_test.go +++ b/federationapi/federationapi_test.go @@ -30,7 +30,7 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) { fsAPI := base.FederationAPIHTTPClient() // TODO: This is pretty fragile, as if anything calls anything on these nils this test will break. // Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing. - federationapi.AddPublicRoutes(base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicWellKnownAPIMux, &cfg.FederationAPI, nil, nil, keyRing, nil, fsAPI, nil, nil, &cfg.MSCs, nil) + federationapi.AddPublicRoutes(base.ProcessContext, base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicWellKnownAPIMux, &cfg.FederationAPI, nil, nil, keyRing, nil, fsAPI, nil, &cfg.MSCs, nil) baseURL, cancel := test.ListenAndServe(t, base.PublicFederationAPIMux, true) defer cancel() serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://")) diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go index b888b3654..8cd944346 100644 --- a/federationapi/internal/perform.go +++ b/federationapi/internal/perform.go @@ -392,17 +392,17 @@ func (r *FederationInternalAPI) performOutboundPeekUsingServer( // we have the peek state now so let's process regardless of whether upstream gives up ctx = context.Background() - respState := respPeek.ToRespState() - authEvents := respState.AuthEvents.UntrustedEvents(respPeek.RoomVersion) + // authenticate the state returned (check its auth events etc) // the equivalent of CheckSendJoinResponse() + authEvents, _, err := respState.Check(ctx, respPeek.RoomVersion, r.keyRing, federatedAuthProvider(ctx, r.federation, r.keyRing, serverName)) + if err != nil { + return fmt.Errorf("error checking state returned from peeking: %w", err) + } if err = sanityCheckAuthChain(authEvents); err != nil { return fmt.Errorf("sanityCheckAuthChain: %w", err) } - if err = respState.Check(ctx, respPeek.RoomVersion, r.keyRing, federatedAuthProvider(ctx, r.federation, r.keyRing, serverName)); err != nil { - return fmt.Errorf("error checking state returned from peeking: %w", err) - } // If we've got this far, the remote server is peeking. if renewing { diff --git a/federationapi/producers/syncapi.go b/federationapi/producers/syncapi.go new file mode 100644 index 000000000..494150036 --- /dev/null +++ b/federationapi/producers/syncapi.go @@ -0,0 +1,163 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package producers + +import ( + "context" + "encoding/json" + "strconv" + "time" + + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/syncapi/types" + userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/nats-io/nats.go" + log "github.com/sirupsen/logrus" +) + +// SyncAPIProducer produces events for the sync API server to consume +type SyncAPIProducer struct { + TopicReceiptEvent string + TopicSendToDeviceEvent string + TopicTypingEvent string + TopicPresenceEvent string + JetStream nats.JetStreamContext + ServerName gomatrixserverlib.ServerName + UserAPI userapi.UserInternalAPI +} + +func (p *SyncAPIProducer) SendReceipt( + ctx context.Context, + userID, roomID, eventID, receiptType string, timestamp gomatrixserverlib.Timestamp, +) error { + m := &nats.Msg{ + Subject: p.TopicReceiptEvent, + Header: nats.Header{}, + } + m.Header.Set(jetstream.UserID, userID) + m.Header.Set(jetstream.RoomID, roomID) + m.Header.Set(jetstream.EventID, eventID) + m.Header.Set("type", receiptType) + m.Header.Set("timestamp", strconv.Itoa(int(timestamp))) + + log.WithFields(log.Fields{}).Tracef("Producing to topic '%s'", p.TopicReceiptEvent) + _, err := p.JetStream.PublishMsg(m, nats.Context(ctx)) + return err +} + +func (p *SyncAPIProducer) SendToDevice( + ctx context.Context, sender, userID, deviceID, eventType string, + message interface{}, +) error { + devices := []string{} + _, domain, err := gomatrixserverlib.SplitID('@', userID) + if err != nil { + return err + } + + // If the event is targeted locally then we want to expand the wildcard + // out into individual device IDs so that we can send them to each respective + // device. If the event isn't targeted locally then we can't expand the + // wildcard as we don't know about the remote devices, so instead we leave it + // as-is, so that the federation sender can send it on with the wildcard intact. + if domain == p.ServerName && deviceID == "*" { + var res userapi.QueryDevicesResponse + err = p.UserAPI.QueryDevices(context.TODO(), &userapi.QueryDevicesRequest{ + UserID: userID, + }, &res) + if err != nil { + return err + } + for _, dev := range res.Devices { + devices = append(devices, dev.ID) + } + } else { + devices = append(devices, deviceID) + } + + js, err := json.Marshal(message) + if err != nil { + return err + } + + log.WithFields(log.Fields{ + "user_id": userID, + "num_devices": len(devices), + "type": eventType, + }).Tracef("Producing to topic '%s'", p.TopicSendToDeviceEvent) + for _, device := range devices { + ote := &types.OutputSendToDeviceEvent{ + UserID: userID, + DeviceID: device, + SendToDeviceEvent: gomatrixserverlib.SendToDeviceEvent{ + Sender: sender, + Type: eventType, + Content: js, + }, + } + + eventJSON, err := json.Marshal(ote) + if err != nil { + log.WithError(err).Error("sendToDevice failed json.Marshal") + return err + } + m := &nats.Msg{ + Subject: p.TopicSendToDeviceEvent, + Data: eventJSON, + Header: nats.Header{}, + } + m.Header.Set("sender", sender) + m.Header.Set(jetstream.UserID, userID) + + if _, err = p.JetStream.PublishMsg(m, nats.Context(ctx)); err != nil { + log.WithError(err).Error("sendToDevice failed t.Producer.SendMessage") + return err + } + } + return nil +} + +func (p *SyncAPIProducer) SendTyping( + ctx context.Context, userID, roomID string, typing bool, timeoutMS int64, +) error { + m := &nats.Msg{ + Subject: p.TopicTypingEvent, + Header: nats.Header{}, + } + m.Header.Set(jetstream.UserID, userID) + m.Header.Set(jetstream.RoomID, roomID) + m.Header.Set("typing", strconv.FormatBool(typing)) + m.Header.Set("timeout_ms", strconv.Itoa(int(timeoutMS))) + _, err := p.JetStream.PublishMsg(m, nats.Context(ctx)) + return err +} + +func (p *SyncAPIProducer) SendPresence( + ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveAgo int64, +) error { + m := nats.NewMsg(p.TopicPresenceEvent) + m.Header.Set(jetstream.UserID, userID) + m.Header.Set("presence", presence.String()) + if statusMsg != nil { + m.Header.Set("status_msg", *statusMsg) + } + lastActiveTS := gomatrixserverlib.AsTimestamp(time.Now().Add(-(time.Duration(lastActiveAgo) * time.Millisecond))) + + m.Header.Set("last_active_ts", strconv.Itoa(int(lastActiveTS))) + log.Debugf("Sending presence to syncAPI: %+v", m.Header) + _, err := p.JetStream.PublishMsg(m, nats.Context(ctx)) + return err +} diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go index dcd090856..5b5481274 100644 --- a/federationapi/queue/queue.go +++ b/federationapi/queue/queue.go @@ -104,28 +104,31 @@ func NewOutgoingQueues( } // Look up which servers we have pending items for and then rehydrate those queues. if !disabled { - time.AfterFunc(time.Second*5, func() { - serverNames := map[gomatrixserverlib.ServerName]struct{}{} - if names, err := db.GetPendingPDUServerNames(context.Background()); err == nil { - for _, serverName := range names { - serverNames[serverName] = struct{}{} - } - } else { - log.WithError(err).Error("Failed to get PDU server names for destination queue hydration") + serverNames := map[gomatrixserverlib.ServerName]struct{}{} + if names, err := db.GetPendingPDUServerNames(context.Background()); err == nil { + for _, serverName := range names { + serverNames[serverName] = struct{}{} } - if names, err := db.GetPendingEDUServerNames(context.Background()); err == nil { - for _, serverName := range names { - serverNames[serverName] = struct{}{} - } - } else { - log.WithError(err).Error("Failed to get EDU server names for destination queue hydration") + } else { + log.WithError(err).Error("Failed to get PDU server names for destination queue hydration") + } + if names, err := db.GetPendingEDUServerNames(context.Background()); err == nil { + for _, serverName := range names { + serverNames[serverName] = struct{}{} } - for serverName := range serverNames { - if queue := queues.getQueue(serverName); queue != nil { - queue.wakeQueueIfNeeded() - } + } else { + log.WithError(err).Error("Failed to get EDU server names for destination queue hydration") + } + offset, step := time.Second*5, time.Second + if max := len(serverNames); max > 120 { + step = (time.Second * 120) / time.Duration(max) + } + for serverName := range serverNames { + if queue := queues.getQueue(serverName); queue != nil { + time.AfterFunc(offset, queue.wakeQueueIfNeeded) + offset += step } - }) + } } return queues } diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 04c88d957..a085ed780 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -19,8 +19,8 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/clientapi/jsonerror" - eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" federationAPI "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/federationapi/producers" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/httputil" keyserverAPI "github.com/matrix-org/dendrite/keyserver/api" @@ -29,6 +29,7 @@ import ( userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" + "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" ) @@ -44,7 +45,6 @@ func Setup( fedMux, keyMux, wkMux *mux.Router, cfg *config.FederationAPI, rsAPI roomserverAPI.RoomserverInternalAPI, - eduAPI eduserverAPI.EDUServerInputAPI, fsAPI federationAPI.FederationInternalAPI, keys gomatrixserverlib.JSONVerifier, federation *gomatrixserverlib.FederationClient, @@ -52,7 +52,12 @@ func Setup( keyAPI keyserverAPI.KeyInternalAPI, mscCfg *config.MSCs, servers federationAPI.ServersInRoomProvider, + producer *producers.SyncAPIProducer, ) { + prometheus.MustRegister( + pduCountTotal, eduCountTotal, + ) + v2keysmux := keyMux.PathPrefix("/v2").Subrouter() v1fedmux := fedMux.PathPrefix("/v1").Subrouter() v2fedmux := fedMux.PathPrefix("/v2").Subrouter() @@ -116,7 +121,7 @@ func Setup( func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return Send( httpReq, request, gomatrixserverlib.TransactionID(vars["txnID"]), - cfg, rsAPI, eduAPI, keyAPI, keys, federation, mu, servers, + cfg, rsAPI, keyAPI, keys, federation, mu, servers, producer, ) }, )).Methods(http.MethodPut, http.MethodOptions) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 745e36de9..f2b902b6f 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -23,12 +23,14 @@ import ( "time" "github.com/matrix-org/dendrite/clientapi/jsonerror" - eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" federationAPI "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/federationapi/producers" + "github.com/matrix-org/dendrite/federationapi/types" "github.com/matrix-org/dendrite/internal" keyapi "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" + syncTypes "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/prometheus/client_golang/prometheus" @@ -72,12 +74,6 @@ var ( ) ) -func init() { - prometheus.MustRegister( - pduCountTotal, eduCountTotal, - ) -} - var inFlightTxnsPerOrigin sync.Map // transaction ID -> chan util.JSONResponse // Send implements /_matrix/federation/v1/send/{txnID} @@ -87,12 +83,12 @@ func Send( txnID gomatrixserverlib.TransactionID, cfg *config.FederationAPI, rsAPI api.RoomserverInternalAPI, - eduAPI eduserverAPI.EDUServerInputAPI, keyAPI keyapi.KeyInternalAPI, keys gomatrixserverlib.JSONVerifier, federation *gomatrixserverlib.FederationClient, mu *internal.MutexByRoom, servers federationAPI.ServersInRoomProvider, + producer *producers.SyncAPIProducer, ) util.JSONResponse { // First we should check if this origin has already submitted this // txn ID to us. If they have and the txnIDs map contains an entry, @@ -126,13 +122,14 @@ func Send( defer inFlightTxnsPerOrigin.Delete(index) t := txnReq{ - rsAPI: rsAPI, - eduAPI: eduAPI, - keys: keys, - federation: federation, - servers: servers, - keyAPI: keyAPI, - roomsMu: mu, + rsAPI: rsAPI, + keys: keys, + federation: federation, + servers: servers, + keyAPI: keyAPI, + roomsMu: mu, + producer: producer, + inboundPresenceEnabled: cfg.Matrix.Presence.EnableInbound, } var txnEvents struct { @@ -184,13 +181,14 @@ func Send( type txnReq struct { gomatrixserverlib.Transaction - rsAPI api.RoomserverInternalAPI - eduAPI eduserverAPI.EDUServerInputAPI - keyAPI keyapi.KeyInternalAPI - keys gomatrixserverlib.JSONVerifier - federation txnFederationClient - roomsMu *internal.MutexByRoom - servers federationAPI.ServersInRoomProvider + rsAPI api.RoomserverInternalAPI + keyAPI keyapi.KeyInternalAPI + keys gomatrixserverlib.JSONVerifier + federation txnFederationClient + roomsMu *internal.MutexByRoom + servers federationAPI.ServersInRoomProvider + producer *producers.SyncAPIProducer + inboundPresenceEnabled bool } // A subset of FederationClient functionality that txn requires. Useful for testing. @@ -329,8 +327,8 @@ func (t *txnReq) processEDUs(ctx context.Context) { util.GetLogger(ctx).Debugf("Dropping typing event where sender domain (%q) doesn't match origin (%q)", domain, t.Origin) continue } - if err := eduserverAPI.SendTyping(ctx, t.eduAPI, typingPayload.UserID, typingPayload.RoomID, typingPayload.Typing, 30*1000); err != nil { - util.GetLogger(ctx).WithError(err).Error("Failed to send typing event to edu server") + if err := t.producer.SendTyping(ctx, typingPayload.UserID, typingPayload.RoomID, typingPayload.Typing, 30*1000); err != nil { + util.GetLogger(ctx).WithError(err).Error("Failed to send typing event to JetStream") } case gomatrixserverlib.MDirectToDevice: // https://matrix.org/docs/spec/server_server/r0.1.3#m-direct-to-device-schema @@ -342,12 +340,12 @@ func (t *txnReq) processEDUs(ctx context.Context) { for userID, byUser := range directPayload.Messages { for deviceID, message := range byUser { // TODO: check that the user and the device actually exist here - if err := eduserverAPI.SendToDevice(ctx, t.eduAPI, directPayload.Sender, userID, deviceID, directPayload.Type, message); err != nil { + if err := t.producer.SendToDevice(ctx, directPayload.Sender, userID, deviceID, directPayload.Type, message); err != nil { util.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{ "sender": directPayload.Sender, "user_id": userID, "device_id": deviceID, - }).Error("Failed to send send-to-device event to edu server") + }).Error("Failed to send send-to-device event to JetStream") } } } @@ -355,7 +353,7 @@ func (t *txnReq) processEDUs(ctx context.Context) { t.processDeviceListUpdate(ctx, e) case gomatrixserverlib.MReceipt: // https://matrix.org/docs/spec/server_server/r0.1.4#receipts - payload := map[string]eduserverAPI.FederationReceiptMRead{} + payload := map[string]types.FederationReceiptMRead{} if err := json.Unmarshal(e.Content, &payload); err != nil { util.GetLogger(ctx).WithError(err).Debug("Failed to unmarshal receipt event") @@ -379,23 +377,47 @@ func (t *txnReq) processEDUs(ctx context.Context) { "user_id": userID, "room_id": roomID, "events": mread.EventIDs, - }).Error("Failed to send receipt event to edu server") + }).Error("Failed to send receipt event to JetStream") continue } } } - case eduserverAPI.MSigningKeyUpdate: + case types.MSigningKeyUpdate: if err := t.processSigningKeyUpdate(ctx, e); err != nil { logrus.WithError(err).Errorf("Failed to process signing key update") } + case gomatrixserverlib.MPresence: + if t.inboundPresenceEnabled { + if err := t.processPresence(ctx, e); err != nil { + logrus.WithError(err).Errorf("Failed to process presence update") + } + } default: util.GetLogger(ctx).WithField("type", e.Type).Debug("Unhandled EDU") } } } +// processPresence handles m.receipt events +func (t *txnReq) processPresence(ctx context.Context, e gomatrixserverlib.EDU) error { + payload := types.Presence{} + if err := json.Unmarshal(e.Content, &payload); err != nil { + return err + } + for _, content := range payload.Push { + presence, ok := syncTypes.PresenceFromString(content.Presence) + if !ok { + continue + } + if err := t.producer.SendPresence(ctx, content.UserID, presence, content.StatusMsg, content.LastActiveAgo); err != nil { + return err + } + } + return nil +} + func (t *txnReq) processSigningKeyUpdate(ctx context.Context, e gomatrixserverlib.EDU) error { - var updatePayload eduserverAPI.CrossSigningKeyUpdate + var updatePayload keyapi.CrossSigningKeyUpdate if err := json.Unmarshal(e.Content, &updatePayload); err != nil { util.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{ "user_id": updatePayload.UserID, @@ -422,7 +444,7 @@ func (t *txnReq) processSigningKeyUpdate(ctx context.Context, e gomatrixserverli return nil } -// processReceiptEvent sends receipt events to the edu server +// processReceiptEvent sends receipt events to JetStream func (t *txnReq) processReceiptEvent(ctx context.Context, userID, roomID, receiptType string, timestamp gomatrixserverlib.Timestamp, @@ -430,17 +452,7 @@ func (t *txnReq) processReceiptEvent(ctx context.Context, ) error { // store every event for _, eventID := range eventIDs { - req := eduserverAPI.InputReceiptEventRequest{ - InputReceiptEvent: eduserverAPI.InputReceiptEvent{ - UserID: userID, - RoomID: roomID, - EventID: eventID, - Type: receiptType, - Timestamp: timestamp, - }, - } - resp := eduserverAPI.InputReceiptEventResponse{} - if err := t.eduAPI.InputReceiptEvent(ctx, &req, &resp); err != nil { + if err := t.producer.SendReceipt(ctx, userID, roomID, eventID, receiptType, timestamp); err != nil { return fmt.Errorf("unable to set receipt event: %w", err) } } diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index 4280643e9..8d2d85040 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -7,7 +7,6 @@ import ( "testing" "time" - eduAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/test" "github.com/matrix-org/dendrite/roomserver/api" @@ -53,44 +52,6 @@ func init() { } } -type testEDUProducer struct { - // this producer keeps track of calls to InputTypingEvent - invocations []eduAPI.InputTypingEventRequest -} - -func (p *testEDUProducer) InputTypingEvent( - ctx context.Context, - request *eduAPI.InputTypingEventRequest, - response *eduAPI.InputTypingEventResponse, -) error { - p.invocations = append(p.invocations, *request) - return nil -} - -func (p *testEDUProducer) InputSendToDeviceEvent( - ctx context.Context, - request *eduAPI.InputSendToDeviceEventRequest, - response *eduAPI.InputSendToDeviceEventResponse, -) error { - return nil -} - -func (o *testEDUProducer) InputReceiptEvent( - ctx context.Context, - request *eduAPI.InputReceiptEventRequest, - response *eduAPI.InputReceiptEventResponse, -) error { - return nil -} - -func (o *testEDUProducer) InputCrossSigningKeyUpdate( - ctx context.Context, - request *eduAPI.InputCrossSigningKeyUpdateRequest, - response *eduAPI.InputCrossSigningKeyUpdateResponse, -) error { - return nil -} - type testRoomserverAPI struct { api.RoomserverInternalAPITrace inputRoomEvents []api.InputRoomEvent @@ -225,7 +186,6 @@ func (c *txnFedClient) LookupMissingEvents(ctx context.Context, s gomatrixserver func mustCreateTransaction(rsAPI api.RoomserverInternalAPI, fedClient txnFederationClient, pdus []json.RawMessage) *txnReq { t := &txnReq{ rsAPI: rsAPI, - eduAPI: &testEDUProducer{}, keys: &test.NopJSONVerifier{}, federation: fedClient, roomsMu: internal.NewMutexByRoom(), diff --git a/federationapi/types/types.go b/federationapi/types/types.go index c486c05c4..5821000cc 100644 --- a/federationapi/types/types.go +++ b/federationapi/types/types.go @@ -18,6 +18,8 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) +const MSigningKeyUpdate = "m.signing_key_update" // TODO: move to gomatrixserverlib + // A JoinedHost is a server that is joined to a matrix room. type JoinedHost struct { // The MemberEventID of a m.room.member join event. @@ -51,3 +53,28 @@ type InboundPeek struct { RenewedTimestamp int64 RenewalInterval int64 } + +type FederationReceiptMRead struct { + User map[string]FederationReceiptData `json:"m.read"` +} + +type FederationReceiptData struct { + Data ReceiptTS `json:"data"` + EventIDs []string `json:"event_ids"` +} + +type ReceiptTS struct { + TS gomatrixserverlib.Timestamp `json:"ts"` +} + +type Presence struct { + Push []PresenceContent `json:"push"` +} + +type PresenceContent struct { + CurrentlyActive bool `json:"currently_active,omitempty"` + LastActiveAgo int64 `json:"last_active_ago"` + Presence string `json:"presence"` + StatusMsg *string `json:"status_msg,omitempty"` + UserID string `json:"user_id"` +} diff --git a/go.mod b/go.mod index ae925c719..f4ac8d123 100644 --- a/go.mod +++ b/go.mod @@ -12,20 +12,20 @@ require ( github.com/MFAshby/stdemuxerhook v1.0.0 github.com/Masterminds/semver/v3 v3.1.1 github.com/codeclysm/extract v2.2.0+incompatible - github.com/containerd/containerd v1.5.9 // indirect - github.com/docker/docker v20.10.12+incompatible + github.com/containerd/containerd v1.6.2 // indirect + github.com/docker/docker v20.10.14+incompatible github.com/docker/go-connections v0.4.0 - github.com/frankban/quicktest v1.14.0 // indirect - github.com/getsentry/sentry-go v0.12.0 + github.com/frankban/quicktest v1.14.3 // indirect + github.com/getsentry/sentry-go v0.13.0 github.com/gologme/log v1.3.0 - github.com/google/go-cmp v0.5.6 - github.com/google/uuid v1.2.0 + github.com/google/go-cmp v0.5.7 + github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 - github.com/gorilla/websocket v1.4.2 + github.com/gorilla/websocket v1.5.0 github.com/h2non/filetype v1.1.3 // indirect github.com/hashicorp/golang-lru v0.5.4 - github.com/juju/testing v0.0.0-20211215003918-77eb13d6cad2 // indirect - github.com/lib/pq v1.10.4 + github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494 // indirect + github.com/lib/pq v1.10.5 github.com/libp2p/go-libp2p v0.13.0 github.com/libp2p/go-libp2p-circuit v0.4.0 github.com/libp2p/go-libp2p-core v0.8.3 @@ -38,18 +38,16 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20220317164600-0980b7f341e0 - github.com/matrix-org/pinecone v0.0.0-20220323142759-6fb077377278 + github.com/matrix-org/gomatrixserverlib v0.0.0-20220405134050-301e340659d5 + github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.10 - github.com/morikuni/aec v1.0.0 // indirect github.com/nats-io/nats-server/v2 v2.7.4-0.20220309205833-773636c1c5bb github.com/nats-io/nats.go v1.13.1-0.20220308171302-2f2f6968e98d github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 github.com/ngrok/sqlmw v0.0.0-20211220175533-9d16fdc47b31 - github.com/onsi/ginkgo v1.16.4 // indirect - github.com/onsi/gomega v1.13.0 // indirect + github.com/opencontainers/image-spec v1.0.2 // indirect github.com/opentracing/opentracing-go v1.2.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 @@ -62,15 +60,13 @@ require ( github.com/uber/jaeger-lib v2.4.1+incompatible github.com/yggdrasil-network/yggdrasil-go v0.4.3 go.uber.org/atomic v1.9.0 - golang.org/x/crypto v0.0.0-20220214200702-86341886e292 - golang.org/x/image v0.0.0-20211028202545-6944b10bf410 - golang.org/x/mobile v0.0.0-20220112015953-858099ff7816 - golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd - golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect + golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 + golang.org/x/image v0.0.0-20220321031419-a8550c1d254a + golang.org/x/mobile v0.0.0-20220407111146-e579adbbc4a2 + golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 - gopkg.in/h2non/bimg.v1 v1.1.5 + gopkg.in/h2non/bimg.v1 v1.1.9 gopkg.in/yaml.v2 v2.4.0 - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect nhooyr.io/websocket v1.8.7 ) diff --git a/go.sum b/go.sum index 25174f468..063365168 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,9 @@ bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= +bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= @@ -14,6 +17,11 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -22,6 +30,7 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -31,7 +40,13 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= +dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= +dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= +git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/Arceliar/ironwood v0.0.0-20211125050254-8951369625d0 h1:QUqcb7BOcBU2p7Nax7pESOb8hrZYtI0Ts6j4v4mvcQo= @@ -39,19 +54,23 @@ github.com/Arceliar/ironwood v0.0.0-20211125050254-8951369625d0/go.mod h1:RP72ru github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979 h1:WndgpSW13S32VLQ3ugUxx2EnnWmgba1kCqPkd4Gk1yQ= github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= @@ -73,8 +92,9 @@ github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.4.17 h1:iT12IBVClFevaf8PuVyi3UmZOVh4OqnaLxDTW2O6j3w= github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY= +github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= @@ -82,12 +102,18 @@ github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg3 github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= +github.com/Microsoft/hcsshim v0.8.20/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= +github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= +github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= github.com/RyanCarrier/dijkstra v1.0.0/go.mod h1:5agGUBNEtUAGIANmbw09fuO3a2htPEkc1jNH01qxCWA= @@ -106,6 +132,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= +github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk= github.com/anacrolix/envpprof v0.0.0-20180404065416-323002cec2fa/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c= github.com/anacrolix/envpprof v1.0.0/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c= github.com/anacrolix/envpprof v1.1.1 h1:sHQCyj7HtiSfaZAzL2rJrQdyS7odLqlwO6nhk/tG/j8= @@ -117,7 +144,12 @@ github.com/anacrolix/missinggo v1.2.1 h1:0IE3TqX5y5D0IxeMwTyIgqdDew4QrzcXaaEnJQy github.com/anacrolix/missinggo v1.2.1/go.mod h1:J5cMhif8jPmFoC3+Uvob3OXXNIhOUikzMt+uUjeM21Y= github.com/anacrolix/missinggo/perf v1.0.0/go.mod h1:ljAFWkBuzkO12MQclXzZrosP5urunoLS0Cbvb4V0uMQ= github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= @@ -132,9 +164,11 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo= github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo= github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8= @@ -154,11 +188,16 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -166,6 +205,9 @@ github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cb github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= +github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= +github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= +github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -175,9 +217,20 @@ github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLI github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= +github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= +github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/codeclysm/extract v2.2.0+incompatible h1:q3wyckoA30bhUSiwdQezMqVhwd8+WGE64/GL//LtUhI= github.com/codeclysm/extract v2.2.0+incompatible/go.mod h1:2nhFMPHiU9At61hz+12bfrlpXSUrOnK+wR+KlGO4Uks= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= @@ -195,11 +248,13 @@ github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4S github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= +github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= +github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= @@ -213,8 +268,11 @@ github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7 github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= -github.com/containerd/containerd v1.5.9 h1:rs6Xg1gtIxaeyG+Smsb/0xaSDu1VgFhOCKBXxMxbsF4= -github.com/containerd/containerd v1.5.9/go.mod h1:fvQqCfadDGga5HZyn3j4+dx56qj2I9YwBrlSdalvJYQ= +github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= +github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= +github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s= +github.com/containerd/containerd v1.6.2 h1:pcaPUGbYW8kBw6OgIZwIVIeEhdWVrBzsoCfVJ5BjrLU= +github.com/containerd/containerd v1.6.2/go.mod h1:sidY30/InSE1j2vdD1ihtKoJz+lWdaXMdiAeIupaf+s= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -222,6 +280,7 @@ github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cE github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= +github.com/containerd/continuity v0.2.2/go.mod h1:pWygW9u7LtS1o4N/Tn0FoCFDIXZ7rxcMX7HX1Dmibvk= github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= @@ -230,6 +289,8 @@ github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1S github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU= github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk= +github.com/containerd/go-cni v1.1.0/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA= +github.com/containerd/go-cni v1.1.3/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA= github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= @@ -239,9 +300,11 @@ github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms= +github.com/containerd/imgcrypt v1.1.3/go.mod h1:/TPA1GIDXMzbj01yd8pIbQiLdQxed5ue1wb8bP7PQu4= github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= +github.com/containerd/stargz-snapshotter/estargz v0.4.1/go.mod h1:x7Q9dg9QYb4+ELgxmo4gBUeJB0tl5dqH1Sdz0nJU1QM= github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8= @@ -260,22 +323,28 @@ github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNR github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v1.0.1/go.mod h1:AKuhXbN5EzmD4yTNtfSsX3tPcmtrBI6QcRV0NiNt15Y= github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= +github.com/containernetworking/plugins v1.0.1/go.mod h1:QHCfGpaTwYTbbH+nZXKVTxNBDZcxSOplJT5ico8/FLE= github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= +github.com/containers/ocicrypt v1.1.2/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= +github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= @@ -287,7 +356,9 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= +github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= @@ -310,12 +381,15 @@ github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUn github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v20.10.12+incompatible h1:CEeNmFM0QZIsJCZKMkZx0ZcahTiewkrgiwfYD+dfl1U= -github.com/docker/docker v20.10.12+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.14+incompatible h1:+T9/PRYWNDo5SZl5qS1r9Mo/0Q8AwxKKPtu9S1yxM0w= +github.com/docker/docker v20.10.14+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= @@ -337,39 +411,49 @@ github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= +github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k= github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss= -github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= -github.com/getsentry/sentry-go v0.12.0 h1:era7g0re5iY13bHSdN/xMkyV+5zZppjRVQhZrXCaEIk= -github.com/getsentry/sentry-go v0.12.0/go.mod h1:NSap0JBYWzHND8oMbyi0+XZhUalc1TBdRL1M71JZW2c= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/getsentry/sentry-go v0.13.0 h1:20dgTiUSfxRB/EhMPtxcL9ZEbM1ZdR+W/7f7NWD+xWo= +github.com/getsentry/sentry-go v0.13.0/go.mod h1:EOsfu5ZdvKPfeHYV6pTVQnsjfp30+XA7//UooKNumH0= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= -github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= +github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs= +github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= +github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= @@ -387,22 +471,37 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= +github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= +github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= @@ -415,6 +514,7 @@ github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblf github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -431,8 +531,10 @@ github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -440,6 +542,9 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -456,6 +561,7 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -466,6 +572,7 @@ github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85q github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -474,18 +581,24 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= github.com/google/gopacket v1.1.18 h1:lum7VRA9kdlvBi7/v2p7/zcbkduHaCH/SVVyurs7OpY= github.com/google/gopacket v1.1.18/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -493,61 +606,91 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= +github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hjson/hjson-go v3.1.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk= github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= -github.com/huandu/xstrings v1.2.0 h1:yPeWdRnmynF7p+lLYz0H2tthW9lqhMJrQV/U7yy4wX0= -github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= @@ -555,6 +698,7 @@ github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= @@ -604,6 +748,7 @@ github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7Ua github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= +github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbBFOsPw= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= @@ -618,11 +763,15 @@ github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsj github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= +github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -637,17 +786,20 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/ansiterm v0.0.0-20160907234532-b99631de12cf/go.mod h1:UJSiEoRfvx3hP73CvoARgeLjaIOjybY9vj8PUPPFGeU= +github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a/go.mod h1:UJSiEoRfvx3hP73CvoARgeLjaIOjybY9vj8PUPPFGeU= github.com/juju/clock v0.0.0-20190205081909-9c5c9712527c/go.mod h1:nD0vlnrUjcjJhqN5WuCWZyzfd5AHZAC9/ajvbSx69xA= github.com/juju/cmd v0.0.0-20171107070456-e74f39857ca0/go.mod h1:yWJQHl73rdSX4DHVKGqkAip+huBslxRwS8m9CrOLq18= github.com/juju/collections v0.0.0-20200605021417-0d0ec82b7271/go.mod h1:5XgO71dV1JClcOJE+4dzdn4HrI5LiyKd7PlVG6eZYhY= github.com/juju/errors v0.0.0-20150916125642-1b5e39b83d18/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= -github.com/juju/errors v0.0.0-20200330140219-3fe23663418f h1:MCOvExGLpaSIzLYB4iQXEHP4jYVU6vmzLNQPdMVrxnM= github.com/juju/errors v0.0.0-20200330140219-3fe23663418f/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= +github.com/juju/errors v0.0.0-20220203013757-bd733f3c86b9 h1:EJHbsNpQyupmMeWTq7inn+5L/WZ7JfzCVPJ+DP9McCQ= +github.com/juju/errors v0.0.0-20220203013757-bd733f3c86b9/go.mod h1:TRm7EVGA3mQOqSVcBySRY7a9Y1/gyVhh/WTCnc5sD4U= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/juju/httpprof v0.0.0-20141217160036-14bf14c30767/go.mod h1:+MaLYz4PumRkkyHYeXJ2G5g5cIW0sli2bOfpmbaMV/g= github.com/juju/loggo v0.0.0-20170605014607-8232ab8918d9/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= -github.com/juju/loggo v0.0.0-20200526014432-9ce3a2e09b5e h1:FdDd7bdI6cjq5vaoYlK1mfQYfF9sF2VZw8VEZMsl5t8= github.com/juju/loggo v0.0.0-20200526014432-9ce3a2e09b5e/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= +github.com/juju/loggo v0.0.0-20210728185423-eebad3a902c4 h1:NO5tuyw++EGLnz56Q8KMyDZRwJwWO8jQnj285J3FOmY= +github.com/juju/loggo v0.0.0-20210728185423-eebad3a902c4/go.mod h1:NIXFioti1SmKAlKNuUwbMenNdef59IF52+ZzuOmHYkg= github.com/juju/mgo/v2 v2.0.0-20210302023703-70d5d206e208 h1:/WiCm+Vpj87e4QWuWwPD/bNE9kDrWCLvPBHOQNcG2+A= github.com/juju/mgo/v2 v2.0.0-20210302023703-70d5d206e208/go.mod h1:0OChplkvPTZ174D2FYZXg4IB9hbEwyHkD+zT+/eK+Fg= github.com/juju/mutex v0.0.0-20171110020013-1fe2a4bf0a3a/go.mod h1:Y3oOzHH8CQ0Ppt0oCKJ2JFO81/EsWenH5AEqigLH+yY= @@ -655,19 +807,22 @@ github.com/juju/retry v0.0.0-20151029024821-62c620325291/go.mod h1:OohPQGsr4pnxw github.com/juju/retry v0.0.0-20180821225755-9058e192b216/go.mod h1:OohPQGsr4pnxwD5YljhQ+TZnuVRYpa5irjugL1Yuif4= github.com/juju/testing v0.0.0-20180402130637-44801989f0f7/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/juju/testing v0.0.0-20190723135506-ce30eb24acd2/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= -github.com/juju/testing v0.0.0-20211215003918-77eb13d6cad2 h1:CGabFjAad6prZAYtDsF0OrF8Uqn5VdN8KMa5mH9B5fA= -github.com/juju/testing v0.0.0-20211215003918-77eb13d6cad2/go.mod h1:7lxZW0B50+xdGFkvhAb8bwAGt6IU87JB1H9w4t8MNVM= +github.com/juju/testing v0.0.0-20210302031854-2c7ee8570c07/go.mod h1:7lxZW0B50+xdGFkvhAb8bwAGt6IU87JB1H9w4t8MNVM= +github.com/juju/testing v0.0.0-20220202055744-1ad0816210a6/go.mod h1:QgWc2UdIPJ8t3rnvv95tFNOsQDfpXYEZDbP281o3b2c= +github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494 h1:XEDzpuZb8Ma7vLja3+5hzUqVTvAqm5Y+ygvnDs5iTMM= +github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494/go.mod h1:rUquetT0ALL48LHZhyRGvjjBH8xZaZ8dFClulKK5wK4= github.com/juju/utils v0.0.0-20180424094159-2000ea4ff043/go.mod h1:6/KLg8Wz/y2KVGWEpkK9vMNGkOnu4k/cqs8Z1fKjTOk= github.com/juju/utils v0.0.0-20200116185830-d40c2fe10647/go.mod h1:6/KLg8Wz/y2KVGWEpkK9vMNGkOnu4k/cqs8Z1fKjTOk= github.com/juju/utils/v2 v2.0.0-20200923005554-4646bfea2ef1/go.mod h1:fdlDtQlzundleLLz/ggoYinEt/LmnrpNKcNTABQATNI= +github.com/juju/utils/v3 v3.0.0-20220130232349-cd7ecef0e94a/go.mod h1:LzwbbEN7buYjySp4nqnti6c6olSqRXUk6RkbSUUP1n8= github.com/juju/version v0.0.0-20161031051906-1f41e27e54f2/go.mod h1:kE8gK5X0CImdr7qpSKl3xB2PmpySSmfj7zVbkZFs81U= github.com/juju/version v0.0.0-20180108022336-b64dbd566305/go.mod h1:kE8gK5X0CImdr7qpSKl3xB2PmpySSmfj7zVbkZFs81U= github.com/juju/version v0.0.0-20191219164919-81c1be00b9a6/go.mod h1:kE8gK5X0CImdr7qpSKl3xB2PmpySSmfj7zVbkZFs81U= +github.com/juju/version/v2 v2.0.0-20211007103408-2e8da085dc23/go.mod h1:Ljlbryh9sYaUSGXucslAEDf0A2XUSGvDbHJgW8ps6nc= github.com/julienschmidt/httprouter v1.1.1-0.20151013225520-77a895ad01eb/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= -github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/kardianos/minwinsvc v1.0.0/go.mod h1:Bgd0oc+D0Qo3bBytmNtyRKVlp85dAloLKhfxanPFFRc= github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8= @@ -700,6 +855,7 @@ github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -708,8 +864,8 @@ github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awS github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= -github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.5 h1:J+gdV2cUmX7ZqL2B0lFcW0m+egaHC2V3lpO8nWxyYiQ= +github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= github.com/libp2p/go-addr-util v0.0.2 h1:7cWK5cdA5x72jX0g8iLrQWm5TRJZ6CzGdPEhWj7plWU= github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E= @@ -919,15 +1075,30 @@ github.com/libp2p/go-yamux v1.4.1 h1:P1Fe9vF4th5JOxxgQvfbOHkrGqIZniTLf+ddhZp8YTI github.com/libp2p/go-yamux v1.4.1/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= github.com/libp2p/go-yamux/v2 v2.0.0 h1:vSGhAy5u6iHBq11ZDcyHH4Blcf9xlBhT4WQDoOE90LU= github.com/libp2p/go-yamux/v2 v2.0.0/go.mod h1:NVWira5+sVUIU6tu1JWvaRn1dRnG+cawOJiflsAM+7U= +github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= +github.com/lucas-clemente/quic-go v0.26.0 h1:ALBQXr9UJ8A1LyzvceX4jd9QFsHvlI0RR6BkV16o00A= +github.com/lucas-clemente/quic-go v0.26.0/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI= github.com/lunixbochs/vtclean v0.0.0-20160125035106-4fbf7632a2c6/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= +github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lxn/walk v0.0.0-20210112085537-c389da54e794/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ= github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= +github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= +github.com/marten-seemann/qtls-go1-16 v0.1.5 h1:o9JrYPPco/Nukd/HpOHMHZoBDXQqoNtUCmny98/1uqQ= +github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk= +github.com/marten-seemann/qtls-go1-17 v0.1.1 h1:DQjHPq+aOzUeh9/lixAGunn6rIOQyWChPSI4+hgW7jc= +github.com/marten-seemann/qtls-go1-17 v0.1.1/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s= +github.com/marten-seemann/qtls-go1-18 v0.1.1 h1:qp7p7XXUFL7fpBvSS1sWD+uSqPvzNQK43DH+/qEkj0Y= +github.com/marten-seemann/qtls-go1-18 v0.1.1/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= github.com/masterzen/azure-sdk-for-go v3.2.0-beta.0.20161014135628-ee4f0065d00c+incompatible/go.mod h1:mf8fjOu33zCqxUjuiU3I8S1lJMyEAlH+0F2+M5xl3hE= github.com/masterzen/simplexml v0.0.0-20160608183007-4572e39b1ab9/go.mod h1:kCEbxUJlNDEBNbdQMkPSp6yaKcRXVI6f4ddk8Riv4bc= github.com/masterzen/winrm v0.0.0-20161014151040-7a535cd943fc/go.mod h1:CfZSN7zwz5gJiFhZJz49Uzk7mEBHIceWmbFmYx7Hf7E= @@ -941,10 +1112,10 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220317164600-0980b7f341e0 h1:IINbE/0jSYGb7M31StazufyIQdYWSivRlhuns3JYPOM= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220317164600-0980b7f341e0/go.mod h1:+WF5InseAMgi1fTnU46JH39IDpEvLep0fDzx9LDf2Bo= -github.com/matrix-org/pinecone v0.0.0-20220323142759-6fb077377278 h1:lRrvMMv7x1FIVW1mcBdU89lvbgAXKz6RyYR0VQTAr3E= -github.com/matrix-org/pinecone v0.0.0-20220323142759-6fb077377278/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220405134050-301e340659d5 h1:Fkennny7+Z/5pygrhjFMZbz1j++P2hhhLoT7NO3p8DQ= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220405134050-301e340659d5/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48= +github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE= +github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= @@ -955,9 +1126,9 @@ github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.0-20160806122752-66b8e73f3f5c/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -968,6 +1139,8 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= +github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= +github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/mattn/go-sqlite3 v1.14.10 h1:MLn+5bFRlWMGoSRmJour3CL1w/qL96mvipqpwQW/Sfk= github.com/mattn/go-sqlite3 v1.14.10/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= @@ -975,9 +1148,12 @@ github.com/mattomatic/dijkstra v0.0.0-20130617153013-6f6d134eb237/go.mod h1:UOnL github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.25/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.28/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= @@ -995,16 +1171,27 @@ github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1 github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= +github.com/moby/sys/signal v0.6.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= -github.com/moby/term v0.0.0-20200312100748-672ec06f55cd h1:aY7OQNf2XqY/JQ6qREWamhI/81os/agb2BAGpcx5yWI= +github.com/moby/sys/symlink v0.2.0/go.mod h1:7uZVF2dqJjG/NsClqul95CqKOBRQyYSNnJ6BMgR/gFs= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= +github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 h1:yH0SvLzcbZxcJXho2yh7CqdENGMQe73Cw3woZBpPli0= +github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -1087,11 +1274,12 @@ github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= +github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= +github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/neilalexander/nats-server/v2 v2.7.5-0.20220311134712-e2e4a244f30e h1:5tEHLzvDeS6IeqO2o9FFhsE3V2erYj8FlMt2J91wzsk= github.com/neilalexander/nats-server/v2 v2.7.5-0.20220311134712-e2e4a244f30e/go.mod h1:1vZ2Nijh8tcyNe8BDVyTviCd9NYzRbubQYiEHsvOQWc= github.com/neilalexander/nats.go v1.11.1-0.20220104162523-f4ddebe1061c h1:G2qsv7D0rY94HAu8pXmElMluuMHQ85waxIDQBhIzV2Q= github.com/neilalexander/nats.go v1.11.1-0.20220104162523-f4ddebe1061c/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= -github.com/neilalexander/utp v0.1.1-0.20210622132614-ee9a34a30488/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8= github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 h1:lrVQzBtkeQEGGYUHwSX1XPe1E5GL6U3KYCNe2G4bncQ= github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= @@ -1115,6 +1303,8 @@ github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= @@ -1127,8 +1317,9 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= -github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak= github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= +github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU= +github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -1137,6 +1328,7 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.2-0.20211117181255-693428a734f5/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= @@ -1145,6 +1337,7 @@ github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= +github.com/opencontainers/runc v1.1.0/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -1155,14 +1348,18 @@ github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mo github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= +github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= @@ -1174,10 +1371,12 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/pressly/goose v2.7.0+incompatible h1:PWejVEv07LCerQEzMMeAtjuyCKbyprZ/LBa6K5P0OCQ= github.com/pressly/goose v2.7.0+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8= github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -1193,15 +1392,18 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -1218,19 +1420,48 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= +github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= +github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= +github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= +github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= +github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= +github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= +github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= +github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= +github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI= +github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU= +github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag= +github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg= +github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw= +github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y= +github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= +github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ= +github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I= +github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0= +github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ= +github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk= +github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= +github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -1246,6 +1477,9 @@ github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:s github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= +github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= @@ -1259,6 +1493,7 @@ github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKv github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -1267,8 +1502,10 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -1283,10 +1520,12 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w= @@ -1301,6 +1540,8 @@ github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp1 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= @@ -1321,9 +1562,12 @@ github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBn github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= +github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= @@ -1354,11 +1598,11 @@ github.com/yggdrasil-network/yggdrasil-go v0.4.3 h1:LNS7kNpKzFlxQ9xmD5tfmMEvzwa+ github.com/yggdrasil-network/yggdrasil-go v0.4.3/go.mod h1:A1/8kOQT7vzBxlkQtLf1KzJR0cbfL/2zjOCiYOAdjjo= github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= -github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= @@ -1366,43 +1610,83 @@ github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= +go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= +go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= +go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= +go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.28.0/go.mod h1:vEhqr0m4eTc+DWxfsXoXue2GBgV2uUwVznkGIHW/e5w= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= +go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs= +go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0/go.mod h1:hO1KLR7jcKaDDKDkvI9dP/FIhpmna5lkqPUQdEjFAM8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.3.0/go.mod h1:keUU7UfnwWTWpJ+FWnyqmogPa82nuU5VUANFq49hlMY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0/go.mod h1:QNX1aly8ehqqX1LEa6YniTU7VY9I6R3X/oPxhGdTceE= +go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= +go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs= +go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= +go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= +go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v0.11.0/go.mod h1:QpEjXPrNQzrFDZgoTo49dgHR9RYRSrg3NAKnUGl9YpQ= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.0.0 h1:qsup4IcBdlmsnGfqyLl4Ntn3C2XCCuKAE7DwHpScyUo= go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= -go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= +golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180214000028-650f4a345ab4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -1419,13 +1703,15 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE= -golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 h1:tkVvjkPTB7pnW3jnid7kNyAMPVWllTNOf/qKDze4p9o= +golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1443,8 +1729,9 @@ golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMk golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ= -golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20220321031419-a8550c1d254a h1:LnH9RNcpPv5Kzi15lXg42lYMPUf0x8CuPv1YnvBWZAg= +golang.org/x/image v0.0.0-20220321031419-a8550c1d254a/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1454,18 +1741,23 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mobile v0.0.0-20220112015953-858099ff7816 h1:jhDgkcu3yQ4tasBZ+1YwDmK7eFmuVf1w1k+NGGGxfmE= golang.org/x/mobile v0.0.0-20220112015953-858099ff7816/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ= +golang.org/x/mobile v0.0.0-20220325161704-447654d348e3/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ= +golang.org/x/mobile v0.0.0-20220407111146-e579adbbc4a2 h1:XEBytU1NHu2jr/7GWEJBRH3uEhegH+hQcF10Mj/7Cb8= +golang.org/x/mobile v0.0.0-20220407111146-e579adbbc4a2/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180406214816-61147c48b25b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1473,12 +1765,17 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -1512,26 +1809,45 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210927181540-4e4d966f7476/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211011170408-caeb26a5c8c0/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211101193420-4a448f8816b3/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3 h1:EN5+DfgmRMvRUrMGERW2gQl3Vc+Z7ZMnI/xdEpPSf0c= +golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1544,9 +1860,12 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1556,6 +1875,7 @@ golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1604,6 +1924,7 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1612,9 +1933,12 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1622,26 +1946,43 @@ golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211102192858-4dd72447c267/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64 h1:D1v9ucDTYBtbz5vNuBbAhIMAGhQhJ6Ym5ah3maMVNX4= +golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1650,6 +1991,7 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b h1:NXqSWXSRUSCaFuvitrWtU169I3876zRTalMRbfd6LL0= @@ -1658,13 +2000,19 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 h1:GZokNIeuVkl3aZHJchRrr13WCsols02MLUcz1U9is6M= golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1684,6 +2032,7 @@ golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1691,6 +2040,7 @@ golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1711,15 +2061,27 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098 h1:YuekqPskqwCCPM79F1X5Dhv4ezTCj+Ki1oNwiafxkA0= golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= @@ -1737,6 +2099,9 @@ gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1753,15 +2118,26 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= +google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1784,17 +2160,37 @@ google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a h1:pOwg4OoaRYScjmR4LlLgdtnyoHYTSAVhhqe5uPdpII8= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa h1:I0YcKz0I7OAhddo7ya8kMnvprhcWM045PmkBdMO9zN0= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1812,8 +2208,18 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.2 h1:EQyQC3sa8M+p6Ulc8yy9SWSS2GVwyRc83gAbG8lrl4o= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.43.0 h1:Eeu7bZtDZ2DpRCsLhUlcrLnvYaMK1Gz86a+hMVvELmM= +google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1844,14 +2250,13 @@ gopkg.in/errgo.v1 v1.0.0-20161222125816-442357a80af5/go.mod h1:u0ALmqvLRxLI95fkd gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= -gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= -gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= -gopkg.in/h2non/bimg.v1 v1.1.5 h1:w0KZvuIj2FQR0GedFZJNbISxk+oO5tDpkSLi9nuIrfQ= -gopkg.in/h2non/bimg.v1 v1.1.5/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= +gopkg.in/h2non/bimg.v1 v1.1.9 h1:wZIUbeOnwr37Ta4aofhIv8OI8v4ujpjXC9mXnAGpQjM= +gopkg.in/h2non/bimg.v1 v1.1.9/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= gopkg.in/h2non/gock.v1 v1.0.14 h1:fTeu9fcUvSnLNacYvYI54h+1/XEteDyHvrVCZEEEYNM= gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= gopkg.in/httprequest.v1 v1.1.1/go.mod h1:/CkavNL+g3qLOrpFHVrEx4NKepeqR4XTZWNj4sGGjz0= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/macaroon.v2 v2.1.0 h1:HZcsjBCzq9t0eBPMKqTN/uSN6JOm78ZJ2INbqcBQOUI= gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o= @@ -1872,6 +2277,7 @@ gopkg.in/yaml.v2 v2.0.0-20170712054546-1be3d31502d6/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1888,39 +2294,58 @@ gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= +k8s.io/api v0.22.5/go.mod h1:mEhXyLaSD1qTOf40rRiKXkc+2iCem09rWLlFwhCEiAs= k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= +k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= +k8s.io/apimachinery v0.22.5/go.mod h1:xziclGKwuuJ2RM5/rSFQSYAj0zdbci3DH8kj+WvyN0U= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= +k8s.io/apiserver v0.22.5/go.mod h1:s2WbtgZAkTKt679sYtSudEQrTGWUSQAPe6MupLnlmaQ= k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= +k8s.io/client-go v0.22.5/go.mod h1:cs6yf/61q2T1SdQL5Rdcjg9J1ElXSwbjSrW2vFImM4Y= +k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0= k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= +k8s.io/component-base v0.22.5/go.mod h1:VK3I+TjuF9eaa+Ln67dKxhGar5ynVbwnGrUiNF4MqCI= k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc= +k8s.io/cri-api v0.23.1/go.mod h1:REJE3PSU0h/LOV1APBrupxrEJqnoxZC8KWzkBUHwrK4= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= +k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= +k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM= launchpad.net/xmlpath v0.0.0-20130614043138-000000000004/go.mod h1:vqyExLOM3qBx7mvYRkoxjSCF945s0mbe7YynlKYXtsA= nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= @@ -1931,7 +2356,12 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= +sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/eduserver/cache/cache.go b/internal/caching/cache_typing.go similarity index 97% rename from eduserver/cache/cache.go rename to internal/caching/cache_typing.go index f637d7c97..bd6a5fc1b 100644 --- a/eduserver/cache/cache.go +++ b/internal/caching/cache_typing.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cache +package caching import ( "sync" @@ -53,8 +53,8 @@ func (t *EDUCache) newRoomData() *roomData { } } -// New returns a new EDUCache initialised for use. -func New() *EDUCache { +// NewTypingCache returns a new EDUCache initialised for use. +func NewTypingCache() *EDUCache { return &EDUCache{data: make(map[string]*roomData)} } diff --git a/eduserver/cache/cache_test.go b/internal/caching/cache_typing_test.go similarity index 97% rename from eduserver/cache/cache_test.go rename to internal/caching/cache_typing_test.go index c7d01879f..c03d89bc3 100644 --- a/eduserver/cache/cache_test.go +++ b/internal/caching/cache_typing_test.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cache +package caching import ( "testing" @@ -24,9 +24,9 @@ import ( ) func TestEDUCache(t *testing.T) { - tCache := New() + tCache := NewTypingCache() if tCache == nil { - t.Fatal("New failed") + t.Fatal("NewTypingCache failed") } t.Run("AddTypingUser", func(t *testing.T) { diff --git a/internal/eventutil/types.go b/internal/eventutil/types.go index 17861d6c5..afc62d8c2 100644 --- a/internal/eventutil/types.go +++ b/internal/eventutil/types.go @@ -17,6 +17,8 @@ package eventutil import ( "errors" "strconv" + + "github.com/matrix-org/dendrite/syncapi/types" ) // ErrProfileNoExists is returned when trying to lookup a user's profile that @@ -26,9 +28,10 @@ var ErrProfileNoExists = errors.New("no known profile for given user ID") // AccountData represents account data sent from the client API server to the // sync API server type AccountData struct { - RoomID string `json:"room_id"` - Type string `json:"type"` - ReadMarker *ReadMarkerJSON `json:"read_marker,omitempty"` // optional + RoomID string `json:"room_id"` + Type string `json:"type"` + ReadMarker *ReadMarkerJSON `json:"read_marker,omitempty"` // optional + IgnoredUsers *types.IgnoredUsers `json:"ignored_users,omitempty"` // optional } type ReadMarkerJSON struct { diff --git a/internal/httputil/httpapi.go b/internal/httputil/httpapi.go index 1a37a1eec..5fcacd2ad 100644 --- a/internal/httputil/httpapi.go +++ b/internal/httputil/httpapi.go @@ -169,8 +169,9 @@ func MakeHTMLAPI(metricsName string, f func(http.ResponseWriter, *http.Request) return promhttp.InstrumentHandlerCounter( promauto.NewCounterVec( prometheus.CounterOpts{ - Name: metricsName, - Help: "Total number of http requests for HTML resources", + Name: metricsName, + Help: "Total number of http requests for HTML resources", + Namespace: "dendrite", }, []string{"code"}, ), @@ -201,7 +202,28 @@ func MakeInternalAPI(metricsName string, f func(*http.Request) util.JSONResponse h.ServeHTTP(w, req) } - return http.HandlerFunc(withSpan) + return promhttp.InstrumentHandlerCounter( + promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: metricsName + "_requests_total", + Help: "Total number of internal API calls", + Namespace: "dendrite", + }, + []string{"code"}, + ), + promhttp.InstrumentHandlerResponseSize( + promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "dendrite", + Name: metricsName + "_response_size_bytes", + Help: "A histogram of response sizes for requests.", + Buckets: []float64{200, 500, 900, 1500, 5000, 15000, 50000, 100000}, + }, + []string{}, + ), + http.HandlerFunc(withSpan), + ), + ) } // MakeFedAPI makes an http.Handler that checks matrix federation authentication. diff --git a/internal/test/config.go b/internal/test/config.go index 0372fb9c6..d8e0c4531 100644 --- a/internal/test/config.go +++ b/internal/test/config.go @@ -78,8 +78,6 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con cfg.Global.ServerName = gomatrixserverlib.ServerName(assignAddress()) cfg.Global.PrivateKeyPath = config.Path(serverKeyPath) - cfg.FederationAPI.FederationCertificatePaths = []config.Path{config.Path(tlsCertPath)} - cfg.MediaAPI.BasePath = config.Path(mediaBasePath) cfg.Global.JetStream.Addresses = []string{kafkaURI} @@ -97,7 +95,6 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(database) cfg.AppServiceAPI.InternalAPI.Listen = assignAddress() - cfg.EDUServer.InternalAPI.Listen = assignAddress() cfg.FederationAPI.InternalAPI.Listen = assignAddress() cfg.KeyServer.InternalAPI.Listen = assignAddress() cfg.MediaAPI.InternalAPI.Listen = assignAddress() @@ -106,7 +103,6 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con cfg.UserAPI.InternalAPI.Listen = assignAddress() cfg.AppServiceAPI.InternalAPI.Connect = cfg.AppServiceAPI.InternalAPI.Listen - cfg.EDUServer.InternalAPI.Connect = cfg.EDUServer.InternalAPI.Listen cfg.FederationAPI.InternalAPI.Connect = cfg.FederationAPI.InternalAPI.Listen cfg.KeyServer.InternalAPI.Connect = cfg.KeyServer.InternalAPI.Listen cfg.MediaAPI.InternalAPI.Connect = cfg.MediaAPI.InternalAPI.Listen diff --git a/internal/version.go b/internal/version.go index cff4d2819..5227a03bf 100644 --- a/internal/version.go +++ b/internal/version.go @@ -16,8 +16,8 @@ var build string const ( VersionMajor = 0 - VersionMinor = 7 - VersionPatch = 0 + VersionMinor = 8 + VersionPatch = 1 VersionTag = "" // example: "rc1" ) diff --git a/keyserver/api/api.go b/keyserver/api/api.go index d361c6222..429617b10 100644 --- a/keyserver/api/api.go +++ b/keyserver/api/api.go @@ -21,7 +21,6 @@ import ( "strings" "time" - eduapi "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/keyserver/types" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" @@ -66,14 +65,25 @@ const ( // DeviceMessage represents the message produced into Kafka by the key server. type DeviceMessage struct { - Type DeviceMessageType `json:"Type,omitempty"` - *DeviceKeys `json:"DeviceKeys,omitempty"` - *eduapi.OutputCrossSigningKeyUpdate `json:"CrossSigningKeyUpdate,omitempty"` + Type DeviceMessageType `json:"Type,omitempty"` + *DeviceKeys `json:"DeviceKeys,omitempty"` + *OutputCrossSigningKeyUpdate `json:"CrossSigningKeyUpdate,omitempty"` // A monotonically increasing number which represents device changes for this user. StreamID int64 DeviceChangeID int64 } +// OutputCrossSigningKeyUpdate is an entry in the signing key update output kafka log +type OutputCrossSigningKeyUpdate struct { + CrossSigningKeyUpdate `json:"signing_keys"` +} + +type CrossSigningKeyUpdate struct { + MasterKey *gomatrixserverlib.CrossSigningKey `json:"master_key,omitempty"` + SelfSigningKey *gomatrixserverlib.CrossSigningKey `json:"self_signing_key,omitempty"` + UserID string `json:"user_id"` +} + // DeviceKeysEqual returns true if the device keys updates contain the // same display name and key JSON. This will return false if either of // the updates is not a device keys update, or if the user ID/device ID diff --git a/keyserver/internal/cross_signing.go b/keyserver/internal/cross_signing.go index 5124f37e6..0d083b4ba 100644 --- a/keyserver/internal/cross_signing.go +++ b/keyserver/internal/cross_signing.go @@ -22,7 +22,6 @@ import ( "fmt" "strings" - eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/keyserver/types" "github.com/matrix-org/gomatrixserverlib" @@ -246,7 +245,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P } // Finally, generate a notification that we updated the keys. - update := eduserverAPI.CrossSigningKeyUpdate{ + update := api.CrossSigningKeyUpdate{ UserID: req.UserID, } if mk, ok := byPurpose[gomatrixserverlib.CrossSigningKeyPurposeMaster]; ok { @@ -337,7 +336,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceSignatures(ctx context.Context, req for userID := range req.Signatures { masterKey := queryRes.MasterKeys[userID] selfSigningKey := queryRes.SelfSigningKeys[userID] - update := eduserverAPI.CrossSigningKeyUpdate{ + update := api.CrossSigningKeyUpdate{ UserID: userID, MasterKey: &masterKey, SelfSigningKey: &selfSigningKey, diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go index 4b2b8c187..561c9a163 100644 --- a/keyserver/internal/device_list_update.go +++ b/keyserver/internal/device_list_update.go @@ -157,8 +157,15 @@ func (u *DeviceListUpdater) Start() error { if err != nil { return err } + offset, step := time.Second*10, time.Second + if max := len(staleLists); max > 120 { + step = (time.Second * 120) / time.Duration(max) + } for _, userID := range staleLists { - u.notifyWorkers(userID) + time.AfterFunc(offset, func() { + u.notifyWorkers(userID) + }) + offset += step } return nil } diff --git a/keyserver/producers/keychange.go b/keyserver/producers/keychange.go index 9e1c4c645..f86c34177 100644 --- a/keyserver/producers/keychange.go +++ b/keyserver/producers/keychange.go @@ -18,7 +18,6 @@ import ( "context" "encoding/json" - eduapi "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/keyserver/storage" "github.com/matrix-org/dendrite/setup/jetstream" @@ -70,10 +69,10 @@ func (p *KeyChange) ProduceKeyChanges(keys []api.DeviceMessage) error { return nil } -func (p *KeyChange) ProduceSigningKeyUpdate(key eduapi.CrossSigningKeyUpdate) error { +func (p *KeyChange) ProduceSigningKeyUpdate(key api.CrossSigningKeyUpdate) error { output := &api.DeviceMessage{ Type: api.TypeCrossSigningUpdate, - OutputCrossSigningKeyUpdate: &eduapi.OutputCrossSigningKeyUpdate{ + OutputCrossSigningKeyUpdate: &api.OutputCrossSigningKeyUpdate{ CrossSigningKeyUpdate: key, }, } diff --git a/mediaapi/routing/upload.go b/mediaapi/routing/upload.go index ecdab2195..f762b2ff5 100644 --- a/mediaapi/routing/upload.go +++ b/mediaapi/routing/upload.go @@ -169,7 +169,7 @@ func (r *uploadRequest) doUpload( } // Check if temp file size exceeds max file size configuration - if bytesWritten > types.FileSizeBytes(*cfg.MaxFileSizeBytes) { + if *cfg.MaxFileSizeBytes > 0 && bytesWritten > types.FileSizeBytes(*cfg.MaxFileSizeBytes) { fileutils.RemoveDir(tmpDir, r.Logger) // delete temp file return requestEntityTooLargeJSONResponse(*cfg.MaxFileSizeBytes) } diff --git a/mediaapi/routing/upload_test.go b/mediaapi/routing/upload_test.go index 032437b59..e81254f35 100644 --- a/mediaapi/routing/upload_test.go +++ b/mediaapi/routing/upload_test.go @@ -36,6 +36,7 @@ func Test_uploadRequest_doUpload(t *testing.T) { } maxSize := config.FileSizeBytes(8) + unlimitedSize := config.FileSizeBytes(0) logger := log.New().WithField("mediaapi", "test") testdataPath := filepath.Join(wd, "./testdata") @@ -117,6 +118,27 @@ func Test_uploadRequest_doUpload(t *testing.T) { }, want: requestEntityTooLargeJSONResponse(maxSize), }, + { + name: "upload ok with unlimited filesize", + args: args{ + ctx: context.Background(), + reqReader: strings.NewReader("test test test"), + cfg: &config.MediaAPI{ + MaxFileSizeBytes: &unlimitedSize, + BasePath: config.Path(testdataPath), + AbsBasePath: config.Path(testdataPath), + DynamicThumbnails: false, + }, + db: db, + }, + fields: fields{ + Logger: logger, + MediaMetadata: &types.MediaMetadata{ + MediaID: "1339", + UploadName: "test fail", + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/roomserver/api/api.go b/roomserver/api/api.go index bcbf0e4f9..fb77423f8 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -170,6 +170,9 @@ type RoomserverInternalAPI interface { // PerformForget forgets a rooms history for a specific user PerformForget(ctx context.Context, req *PerformForgetRequest, resp *PerformForgetResponse) error + // PerformRoomUpgrade upgrades a room to a newer version + PerformRoomUpgrade(ctx context.Context, req *PerformRoomUpgradeRequest, resp *PerformRoomUpgradeResponse) + // Asks for the default room version as preferred by the server. QueryRoomVersionCapabilities( ctx context.Context, diff --git a/roomserver/api/api_trace.go b/roomserver/api/api_trace.go index 88b372154..ec7211ef8 100644 --- a/roomserver/api/api_trace.go +++ b/roomserver/api/api_trace.go @@ -67,6 +67,15 @@ func (t *RoomserverInternalAPITrace) PerformUnpeek( util.GetLogger(ctx).Infof("PerformUnpeek req=%+v res=%+v", js(req), js(res)) } +func (t *RoomserverInternalAPITrace) PerformRoomUpgrade( + ctx context.Context, + req *PerformRoomUpgradeRequest, + res *PerformRoomUpgradeResponse, +) { + t.Impl.PerformRoomUpgrade(ctx, req, res) + util.GetLogger(ctx).Infof("PerformRoomUpgrade req=%+v res=%+v", js(req), js(res)) +} + func (t *RoomserverInternalAPITrace) PerformJoin( ctx context.Context, req *PerformJoinRequest, diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go index d640858a6..cda4b3ee4 100644 --- a/roomserver/api/perform.go +++ b/roomserver/api/perform.go @@ -203,3 +203,14 @@ type PerformForgetRequest struct { } type PerformForgetResponse struct{} + +type PerformRoomUpgradeRequest struct { + RoomID string `json:"room_id"` + UserID string `json:"user_id"` + RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` +} + +type PerformRoomUpgradeResponse struct { + NewRoomID string + Error *PerformError +} diff --git a/roomserver/api/query.go b/roomserver/api/query.go index 66e85f2f3..8f84edcb5 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -128,6 +128,8 @@ type QueryMembershipForUserResponse struct { type QueryMembershipsForRoomRequest struct { // If true, only returns the membership events of "join" membership JoinedOnly bool `json:"joined_only"` + // If true, only returns the membership events of local users + LocalOnly bool `json:"local_only"` // ID of the room to fetch memberships from RoomID string `json:"room_id"` // Optional - ID of the user sending the request, for checking if the diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go index f96cefcb3..59f485cf7 100644 --- a/roomserver/internal/api.go +++ b/roomserver/internal/api.go @@ -34,6 +34,7 @@ type RoomserverInternalAPI struct { *perform.Publisher *perform.Backfiller *perform.Forgetter + *perform.Upgrader ProcessContext *process.ProcessContext DB storage.Database Cfg *config.RoomServer @@ -159,6 +160,10 @@ func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.FederationInternalA r.Forgetter = &perform.Forgetter{ DB: r.DB, } + r.Upgrader = &perform.Upgrader{ + Cfg: r.Cfg, + URSAPI: r, + } if err := r.Inputer.Start(); err != nil { logrus.WithError(err).Panic("failed to start roomserver input API") diff --git a/roomserver/internal/input/input.go b/roomserver/internal/input/input.go index 6a8ae6d00..1fea6ef06 100644 --- a/roomserver/internal/input/input.go +++ b/roomserver/internal/input/input.go @@ -167,6 +167,7 @@ func (r *Inputer) startWorkerForRoom(roomID string) { // will look to see if we have a worker for that room which has its // own consumer. If we don't, we'll start one. func (r *Inputer) Start() error { + prometheus.MustRegister(roomserverInputBackpressure, processRoomEventDuration) _, err := r.JetStream.Subscribe( "", // This is blank because we specified it in BindStream. func(m *nats.Msg) { @@ -421,10 +422,6 @@ func (r *Inputer) WriteOutputEvents(roomID string, updates []api.OutputEvent) er return nil } -func init() { - prometheus.MustRegister(roomserverInputBackpressure) -} - var roomserverInputBackpressure = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "dendrite", diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index 1acbf0443..3ab9ba4f0 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -37,10 +37,6 @@ import ( "github.com/sirupsen/logrus" ) -func init() { - prometheus.MustRegister(processRoomEventDuration) -} - // TODO: Does this value make sense? const MaximumMissingProcessingTime = time.Minute * 2 diff --git a/roomserver/internal/input/input_missing.go b/roomserver/internal/input/input_missing.go index a7da9b06d..2c958335d 100644 --- a/roomserver/internal/input/input_missing.go +++ b/roomserver/internal/input/input_missing.go @@ -613,12 +613,13 @@ func (t *missingStateReq) lookupMissingStateViaState( return nil, err } // Check that the returned state is valid. - if err := state.Check(ctx, roomVersion, t.keys, nil); err != nil { + authEvents, stateEvents, err := state.Check(ctx, roomVersion, t.keys, nil) + if err != nil { return nil, err } parsedState := &parsedRespState{ - AuthEvents: make([]*gomatrixserverlib.Event, len(state.AuthEvents)), - StateEvents: make([]*gomatrixserverlib.Event, len(state.StateEvents)), + AuthEvents: authEvents, + StateEvents: stateEvents, } // Cache the results of this state lookup and deduplicate anything we already // have in the cache, freeing up memory. diff --git a/roomserver/internal/input/input_test.go b/roomserver/internal/input/input_test.go index 4fa966281..81c86ae38 100644 --- a/roomserver/internal/input/input_test.go +++ b/roomserver/internal/input/input_test.go @@ -2,7 +2,6 @@ package input_test import ( "context" - "fmt" "os" "testing" "time" @@ -12,30 +11,22 @@ import ( "github.com/matrix-org/dendrite/roomserver/internal/input" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/gomatrixserverlib" + "github.com/nats-io/nats.go" ) -func psqlConnectionString() config.DataSource { - user := os.Getenv("POSTGRES_USER") - if user == "" { - user = "dendrite" - } - dbName := os.Getenv("POSTGRES_DB") - if dbName == "" { - dbName = "dendrite" - } - connStr := fmt.Sprintf( - "user=%s dbname=%s sslmode=disable", user, dbName, - ) - password := os.Getenv("POSTGRES_PASSWORD") - if password != "" { - connStr += fmt.Sprintf(" password=%s", password) - } - host := os.Getenv("POSTGRES_HOST") - if host != "" { - connStr += fmt.Sprintf(" host=%s", host) - } - return config.DataSource(connStr) +var js nats.JetStreamContext +var jc *nats.Conn + +func TestMain(m *testing.M) { + var pc *process.ProcessContext + pc, js, jc = jetstream.PrepareForTests() + code := m.Run() + pc.ShutdownDendrite() + pc.WaitForComponentsToFinish() + os.Exit(code) } func TestSingleTransactionOnInput(t *testing.T) { @@ -63,7 +54,7 @@ func TestSingleTransactionOnInput(t *testing.T) { } db, err := storage.Open( &config.DatabaseOptions{ - ConnectionString: psqlConnectionString(), + ConnectionString: "", MaxOpenConnections: 1, MaxIdleConnections: 1, }, @@ -74,7 +65,9 @@ func TestSingleTransactionOnInput(t *testing.T) { t.SkipNow() } inputter := &input.Inputer{ - DB: db, + DB: db, + JetStream: js, + NATSClient: jc, } res := &api.InputRoomEventsResponse{} inputter.InputRoomEvents( diff --git a/roomserver/internal/perform/perform_upgrade.go b/roomserver/internal/perform/perform_upgrade.go new file mode 100644 index 000000000..fcd19b936 --- /dev/null +++ b/roomserver/internal/perform/perform_upgrade.go @@ -0,0 +1,709 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package perform + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/matrix-org/dendrite/internal/eventutil" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/sirupsen/logrus" +) + +type Upgrader struct { + Cfg *config.RoomServer + URSAPI api.RoomserverInternalAPI +} + +// fledglingEvent is a helper representation of an event used when creating many events in succession. +type fledglingEvent struct { + Type string `json:"type"` + StateKey string `json:"state_key"` + Content interface{} `json:"content"` +} + +// PerformRoomUpgrade upgrades a room from one version to another +func (r *Upgrader) PerformRoomUpgrade( + ctx context.Context, + req *api.PerformRoomUpgradeRequest, + res *api.PerformRoomUpgradeResponse, +) { + res.NewRoomID, res.Error = r.performRoomUpgrade(ctx, req) + if res.Error != nil { + res.NewRoomID = "" + logrus.WithContext(ctx).WithError(res.Error).Error("Room upgrade failed") + } +} + +func (r *Upgrader) performRoomUpgrade( + ctx context.Context, + req *api.PerformRoomUpgradeRequest, +) (string, *api.PerformError) { + roomID := req.RoomID + userID := req.UserID + evTime := time.Now() + + // Return an immediate error if the room does not exist + if err := r.validateRoomExists(ctx, roomID); err != nil { + return "", &api.PerformError{ + Code: api.PerformErrorNoRoom, + Msg: "Error validating that the room exists", + } + } + + // 1. Check if the user is authorized to actually perform the upgrade (can send m.room.tombstone) + if !r.userIsAuthorized(ctx, userID, roomID) { + return "", &api.PerformError{ + Code: api.PerformErrorNotAllowed, + Msg: "You don't have permission to upgrade the room, power level too low.", + } + } + + // TODO (#267): Check room ID doesn't clash with an existing one, and we + // probably shouldn't be using pseudo-random strings, maybe GUIDs? + newRoomID := fmt.Sprintf("!%s:%s", util.RandomString(16), r.Cfg.Matrix.ServerName) + + // Get the existing room state for the old room. + oldRoomReq := &api.QueryLatestEventsAndStateRequest{ + RoomID: roomID, + } + oldRoomRes := &api.QueryLatestEventsAndStateResponse{} + if err := r.URSAPI.QueryLatestEventsAndState(ctx, oldRoomReq, oldRoomRes); err != nil { + return "", &api.PerformError{ + Msg: fmt.Sprintf("Failed to get latest state: %s", err), + } + } + + // Make the tombstone event + tombstoneEvent, pErr := r.makeTombstoneEvent(ctx, evTime, userID, roomID, newRoomID) + if pErr != nil { + return "", pErr + } + + // Generate the initial events we need to send into the new room. This includes copied state events and bans + // as well as the power level events needed to set up the room + eventsToMake, pErr := r.generateInitialEvents(ctx, oldRoomRes, userID, roomID, string(req.RoomVersion), tombstoneEvent) + if pErr != nil { + return "", pErr + } + + // 5. Send the tombstone event to the old room (must do this before we set the new canonical_alias) + if pErr = r.sendHeaderedEvent(ctx, tombstoneEvent); pErr != nil { + return "", pErr + } + + // Send the setup events to the new room + if pErr = r.sendInitialEvents(ctx, evTime, userID, newRoomID, string(req.RoomVersion), eventsToMake); pErr != nil { + return "", pErr + } + + // If the old room was public, make sure the new one is too + if pErr = r.publishIfOldRoomWasPublic(ctx, roomID, newRoomID); pErr != nil { + return "", pErr + } + + // If the old room had a canonical alias event, it should be deleted in the old room + if pErr = r.clearOldCanonicalAliasEvent(ctx, oldRoomRes, evTime, userID, roomID); pErr != nil { + return "", pErr + } + + // 4. Move local aliases to the new room + if pErr = moveLocalAliases(ctx, roomID, newRoomID, userID, r.URSAPI); pErr != nil { + return "", pErr + } + + // 6. Restrict power levels in the old room + if pErr = r.restrictOldRoomPowerLevels(ctx, evTime, userID, roomID); pErr != nil { + return "", pErr + } + + return newRoomID, nil +} + +func (r *Upgrader) getRoomPowerLevels(ctx context.Context, roomID string) (*gomatrixserverlib.PowerLevelContent, *api.PerformError) { + oldPowerLevelsEvent := api.GetStateEvent(ctx, r.URSAPI, roomID, gomatrixserverlib.StateKeyTuple{ + EventType: gomatrixserverlib.MRoomPowerLevels, + StateKey: "", + }) + powerLevelContent, err := oldPowerLevelsEvent.PowerLevels() + if err != nil { + util.GetLogger(ctx).WithError(err).Error() + return nil, &api.PerformError{ + Msg: "powerLevel event was not actually a power level event", + } + } + return powerLevelContent, nil +} + +func (r *Upgrader) restrictOldRoomPowerLevels(ctx context.Context, evTime time.Time, userID, roomID string) *api.PerformError { + restrictedPowerLevelContent, pErr := r.getRoomPowerLevels(ctx, roomID) + if pErr != nil { + return pErr + } + + // From: https://spec.matrix.org/v1.2/client-server-api/#server-behaviour-16 + // If possible, the power levels in the old room should also be modified to + // prevent sending of events and inviting new users. For example, setting + // events_default and invite to the greater of 50 and users_default + 1. + restrictedDefaultPowerLevel := int64(50) + if restrictedPowerLevelContent.UsersDefault+1 > restrictedDefaultPowerLevel { + restrictedDefaultPowerLevel = restrictedPowerLevelContent.UsersDefault + 1 + } + restrictedPowerLevelContent.EventsDefault = restrictedDefaultPowerLevel + restrictedPowerLevelContent.Invite = restrictedDefaultPowerLevel + + restrictedPowerLevelsHeadered, resErr := r.makeHeaderedEvent(ctx, evTime, userID, roomID, fledglingEvent{ + Type: gomatrixserverlib.MRoomPowerLevels, + StateKey: "", + Content: restrictedPowerLevelContent, + }) + if resErr != nil { + if resErr.Code == api.PerformErrorNotAllowed { + util.GetLogger(ctx).WithField(logrus.ErrorKey, resErr).Warn("UpgradeRoom: Could not restrict power levels in old room") + } else { + return resErr + } + } else { + if resErr = r.sendHeaderedEvent(ctx, restrictedPowerLevelsHeadered); resErr != nil { + return resErr + } + } + return nil +} + +func moveLocalAliases(ctx context.Context, + roomID, newRoomID, userID string, + URSAPI api.RoomserverInternalAPI) *api.PerformError { + var err error + + aliasReq := api.GetAliasesForRoomIDRequest{RoomID: roomID} + aliasRes := api.GetAliasesForRoomIDResponse{} + if err = URSAPI.GetAliasesForRoomID(ctx, &aliasReq, &aliasRes); err != nil { + return &api.PerformError{ + Msg: "Could not get aliases for old room", + } + } + + for _, alias := range aliasRes.Aliases { + removeAliasReq := api.RemoveRoomAliasRequest{UserID: userID, Alias: alias} + removeAliasRes := api.RemoveRoomAliasResponse{} + if err = URSAPI.RemoveRoomAlias(ctx, &removeAliasReq, &removeAliasRes); err != nil { + return &api.PerformError{ + Msg: "api.RemoveRoomAlias failed", + } + } + + setAliasReq := api.SetRoomAliasRequest{UserID: userID, Alias: alias, RoomID: newRoomID} + setAliasRes := api.SetRoomAliasResponse{} + if err = URSAPI.SetRoomAlias(ctx, &setAliasReq, &setAliasRes); err != nil { + return &api.PerformError{ + Msg: "api.SetRoomAlias failed", + } + } + } + return nil +} + +func (r *Upgrader) clearOldCanonicalAliasEvent(ctx context.Context, oldRoom *api.QueryLatestEventsAndStateResponse, evTime time.Time, userID, roomID string) *api.PerformError { + for _, event := range oldRoom.StateEvents { + if event.Type() != gomatrixserverlib.MRoomCanonicalAlias || !event.StateKeyEquals("") { + continue + } + var aliasContent struct { + Alias string `json:"alias"` + AltAliases []string `json:"alt_aliases"` + } + if err := json.Unmarshal(event.Content(), &aliasContent); err != nil { + return &api.PerformError{ + Msg: fmt.Sprintf("Failed to unmarshal canonical aliases: %s", err), + } + } + if aliasContent.Alias == "" && len(aliasContent.AltAliases) == 0 { + // There are no canonical aliases to clear, therefore do nothing. + return nil + } + } + + emptyCanonicalAliasEvent, resErr := r.makeHeaderedEvent(ctx, evTime, userID, roomID, fledglingEvent{ + Type: gomatrixserverlib.MRoomCanonicalAlias, + Content: map[string]interface{}{}, + }) + if resErr != nil { + if resErr.Code == api.PerformErrorNotAllowed { + util.GetLogger(ctx).WithField(logrus.ErrorKey, resErr).Warn("UpgradeRoom: Could not set empty canonical alias event in old room") + } else { + return resErr + } + } else { + if resErr = r.sendHeaderedEvent(ctx, emptyCanonicalAliasEvent); resErr != nil { + return resErr + } + } + return nil +} + +func (r *Upgrader) publishIfOldRoomWasPublic(ctx context.Context, roomID, newRoomID string) *api.PerformError { + // check if the old room was published + var pubQueryRes api.QueryPublishedRoomsResponse + err := r.URSAPI.QueryPublishedRooms(ctx, &api.QueryPublishedRoomsRequest{ + RoomID: roomID, + }, &pubQueryRes) + if err != nil { + return &api.PerformError{ + Msg: "QueryPublishedRooms failed", + } + } + + // if the old room is published (was public), publish the new room + if len(pubQueryRes.RoomIDs) == 1 { + publishNewRoomAndUnpublishOldRoom(ctx, r.URSAPI, roomID, newRoomID) + } + return nil +} + +func publishNewRoomAndUnpublishOldRoom( + ctx context.Context, + URSAPI api.RoomserverInternalAPI, + oldRoomID, newRoomID string, +) { + // expose this room in the published room list + var pubNewRoomRes api.PerformPublishResponse + URSAPI.PerformPublish(ctx, &api.PerformPublishRequest{ + RoomID: newRoomID, + Visibility: "public", + }, &pubNewRoomRes) + if pubNewRoomRes.Error != nil { + // treat as non-fatal since the room is already made by this point + util.GetLogger(ctx).WithError(pubNewRoomRes.Error).Error("failed to visibility:public") + } + + var unpubOldRoomRes api.PerformPublishResponse + // remove the old room from the published room list + URSAPI.PerformPublish(ctx, &api.PerformPublishRequest{ + RoomID: oldRoomID, + Visibility: "private", + }, &unpubOldRoomRes) + if unpubOldRoomRes.Error != nil { + // treat as non-fatal since the room is already made by this point + util.GetLogger(ctx).WithError(unpubOldRoomRes.Error).Error("failed to visibility:private") + } +} + +func (r *Upgrader) validateRoomExists(ctx context.Context, roomID string) error { + verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} + verRes := api.QueryRoomVersionForRoomResponse{} + if err := r.URSAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil { + return &api.PerformError{ + Code: api.PerformErrorNoRoom, + Msg: "Room does not exist", + } + } + return nil +} + +func (r *Upgrader) userIsAuthorized(ctx context.Context, userID, roomID string, +) bool { + plEvent := api.GetStateEvent(ctx, r.URSAPI, roomID, gomatrixserverlib.StateKeyTuple{ + EventType: gomatrixserverlib.MRoomPowerLevels, + StateKey: "", + }) + if plEvent == nil { + return false + } + pl, err := plEvent.PowerLevels() + if err != nil { + return false + } + // Check for power level required to send tombstone event (marks the current room as obsolete), + // if not found, use the StateDefault power level + return pl.UserLevel(userID) >= pl.EventLevel("m.room.tombstone", true) +} + +// nolint:gocyclo +func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.QueryLatestEventsAndStateResponse, userID, roomID, newVersion string, tombstoneEvent *gomatrixserverlib.HeaderedEvent) ([]fledglingEvent, *api.PerformError) { + state := make(map[gomatrixserverlib.StateKeyTuple]*gomatrixserverlib.HeaderedEvent, len(oldRoom.StateEvents)) + for _, event := range oldRoom.StateEvents { + if event.StateKey() == nil { + // This shouldn't ever happen, but better to be safe than sorry. + continue + } + if event.Type() == gomatrixserverlib.MRoomMember && !event.StateKeyEquals(userID) { + // With the exception of bans and invites which we do want to copy, we + // should ignore membership events that aren't our own, as event auth will + // prevent us from being able to create membership events on behalf of other + // users anyway unless they are invites or bans. + membership, err := event.Membership() + if err != nil { + continue + } + switch membership { + case gomatrixserverlib.Ban: + case gomatrixserverlib.Invite: + default: + continue + } + } + state[gomatrixserverlib.StateKeyTuple{EventType: event.Type(), StateKey: *event.StateKey()}] = event + } + + // The following events are ones that we are going to override manually + // in the following section. + override := map[gomatrixserverlib.StateKeyTuple]struct{}{ + {EventType: gomatrixserverlib.MRoomCreate, StateKey: ""}: {}, + {EventType: gomatrixserverlib.MRoomMember, StateKey: userID}: {}, + {EventType: gomatrixserverlib.MRoomPowerLevels, StateKey: ""}: {}, + {EventType: gomatrixserverlib.MRoomJoinRules, StateKey: ""}: {}, + } + + // The overridden events are essential events that must be present in the + // old room state. Check that they are there. + for tuple := range override { + if _, ok := state[tuple]; !ok { + return nil, &api.PerformError{ + Msg: fmt.Sprintf("Essential event of type %q state key %q is missing", tuple.EventType, tuple.StateKey), + } + } + } + + oldCreateEvent := state[gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomCreate, StateKey: ""}] + oldMembershipEvent := state[gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomMember, StateKey: userID}] + oldPowerLevelsEvent := state[gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomPowerLevels, StateKey: ""}] + oldJoinRulesEvent := state[gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomJoinRules, StateKey: ""}] + + // Create the new room create event. Using a map here instead of CreateContent + // means that we preserve any other interesting fields that might be present + // in the create event (such as for the room types MSC). + newCreateContent := map[string]interface{}{} + _ = json.Unmarshal(oldCreateEvent.Content(), &newCreateContent) + newCreateContent["creator"] = userID + newCreateContent["room_version"] = newVersion + newCreateContent["predecessor"] = gomatrixserverlib.PreviousRoom{ + EventID: tombstoneEvent.EventID(), + RoomID: roomID, + } + newCreateEvent := fledglingEvent{ + Type: gomatrixserverlib.MRoomCreate, + StateKey: "", + Content: newCreateContent, + } + + // Now create the new membership event. Same rules apply as above, so + // that we preserve fields we don't otherwise know about. We'll always + // set the membership to join though, because that is necessary to auth + // the events after it. + newMembershipContent := map[string]interface{}{} + _ = json.Unmarshal(oldMembershipEvent.Content(), &newMembershipContent) + newMembershipContent["membership"] = gomatrixserverlib.Join + newMembershipEvent := fledglingEvent{ + Type: gomatrixserverlib.MRoomMember, + StateKey: userID, + Content: newMembershipContent, + } + + // We might need to temporarily give ourselves a higher power level + // than we had in the old room in order to be able to send all of + // the relevant state events. This function will return whether we + // had to override the power level events or not — if we did, we + // need to send the original power levels again later on. + powerLevelContent, err := oldPowerLevelsEvent.PowerLevels() + if err != nil { + util.GetLogger(ctx).WithError(err).Error() + return nil, &api.PerformError{ + Msg: "Power level event content was invalid", + } + } + tempPowerLevelsEvent, powerLevelsOverridden := createTemporaryPowerLevels(powerLevelContent, userID) + + // Now do the join rules event, same as the create and membership + // events. We'll set a sane default of "invite" so that if the + // existing join rules contains garbage, the room can still be + // upgraded. + newJoinRulesContent := map[string]interface{}{ + "join_rule": gomatrixserverlib.Invite, // sane default + } + _ = json.Unmarshal(oldJoinRulesEvent.Content(), &newJoinRulesContent) + newJoinRulesEvent := fledglingEvent{ + Type: gomatrixserverlib.MRoomJoinRules, + StateKey: "", + Content: newJoinRulesContent, + } + + eventsToMake := make([]fledglingEvent, 0, len(state)) + eventsToMake = append( + eventsToMake, newCreateEvent, newMembershipEvent, + tempPowerLevelsEvent, newJoinRulesEvent, + ) + + // For some reason Sytest expects there to be a guest access event. + // Create one if it doesn't exist. + if _, ok := state[gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomGuestAccess, StateKey: ""}]; !ok { + eventsToMake = append(eventsToMake, fledglingEvent{ + Type: gomatrixserverlib.MRoomGuestAccess, + Content: map[string]string{ + "guest_access": "forbidden", + }, + }) + } + + // Duplicate all of the old state events into the new room. + for tuple, event := range state { + if _, ok := override[tuple]; ok { + // Don't duplicate events we have overridden already. They + // are already in `eventsToMake`. + continue + } + newEvent := fledglingEvent{ + Type: tuple.EventType, + StateKey: tuple.StateKey, + } + if err = json.Unmarshal(event.Content(), &newEvent.Content); err != nil { + logrus.WithError(err).Error("Failed to unmarshal old event") + continue + } + eventsToMake = append(eventsToMake, newEvent) + } + + // If we sent a temporary power level event into the room before, + // override that now by restoring the original power levels. + if powerLevelsOverridden { + eventsToMake = append(eventsToMake, fledglingEvent{ + Type: gomatrixserverlib.MRoomPowerLevels, + Content: powerLevelContent, + }) + } + return eventsToMake, nil +} + +func (r *Upgrader) sendInitialEvents(ctx context.Context, evTime time.Time, userID, newRoomID, newVersion string, eventsToMake []fledglingEvent) *api.PerformError { + var err error + var builtEvents []*gomatrixserverlib.HeaderedEvent + authEvents := gomatrixserverlib.NewAuthEvents(nil) + for i, e := range eventsToMake { + depth := i + 1 // depth starts at 1 + + builder := gomatrixserverlib.EventBuilder{ + Sender: userID, + RoomID: newRoomID, + Type: e.Type, + StateKey: &e.StateKey, + Depth: int64(depth), + } + err = builder.SetContent(e.Content) + if err != nil { + return &api.PerformError{ + Msg: "builder.SetContent failed", + } + } + if i > 0 { + builder.PrevEvents = []gomatrixserverlib.EventReference{builtEvents[i-1].EventReference()} + } + var event *gomatrixserverlib.Event + event, err = r.buildEvent(&builder, &authEvents, evTime, gomatrixserverlib.RoomVersion(newVersion)) + if err != nil { + return &api.PerformError{ + Msg: "buildEvent failed", + } + } + + if err = gomatrixserverlib.Allowed(event, &authEvents); err != nil { + return &api.PerformError{ + Msg: "gomatrixserverlib.Allowed failed", + } + } + + // Add the event to the list of auth events + builtEvents = append(builtEvents, event.Headered(gomatrixserverlib.RoomVersion(newVersion))) + err = authEvents.AddEvent(event) + if err != nil { + return &api.PerformError{ + Msg: "authEvents.AddEvent failed", + } + } + } + + inputs := make([]api.InputRoomEvent, 0, len(builtEvents)) + for _, event := range builtEvents { + inputs = append(inputs, api.InputRoomEvent{ + Kind: api.KindNew, + Event: event, + Origin: r.Cfg.Matrix.ServerName, + SendAsServer: api.DoNotSendToOtherServers, + }) + } + if err = api.SendInputRoomEvents(ctx, r.URSAPI, inputs, false); err != nil { + return &api.PerformError{ + Msg: "api.SendInputRoomEvents failed", + } + } + return nil +} + +func (r *Upgrader) makeTombstoneEvent( + ctx context.Context, + evTime time.Time, + userID, roomID, newRoomID string, +) (*gomatrixserverlib.HeaderedEvent, *api.PerformError) { + content := map[string]interface{}{ + "body": "This room has been replaced", + "replacement_room": newRoomID, + } + event := fledglingEvent{ + Type: "m.room.tombstone", + Content: content, + } + return r.makeHeaderedEvent(ctx, evTime, userID, roomID, event) +} + +func (r *Upgrader) makeHeaderedEvent(ctx context.Context, evTime time.Time, userID, roomID string, event fledglingEvent) (*gomatrixserverlib.HeaderedEvent, *api.PerformError) { + builder := gomatrixserverlib.EventBuilder{ + Sender: userID, + RoomID: roomID, + Type: event.Type, + StateKey: &event.StateKey, + } + err := builder.SetContent(event.Content) + if err != nil { + return nil, &api.PerformError{ + Msg: "builder.SetContent failed", + } + } + var queryRes api.QueryLatestEventsAndStateResponse + headeredEvent, err := eventutil.QueryAndBuildEvent(ctx, &builder, r.Cfg.Matrix, evTime, r.URSAPI, &queryRes) + if err == eventutil.ErrRoomNoExists { + return nil, &api.PerformError{ + Code: api.PerformErrorNoRoom, + Msg: "Room does not exist", + } + } else if e, ok := err.(gomatrixserverlib.BadJSONError); ok { + return nil, &api.PerformError{ + Msg: e.Error(), + } + } else if e, ok := err.(gomatrixserverlib.EventValidationError); ok { + if e.Code == gomatrixserverlib.EventValidationTooLarge { + return nil, &api.PerformError{ + Msg: e.Error(), + } + } + return nil, &api.PerformError{ + Msg: e.Error(), + } + } else if err != nil { + return nil, &api.PerformError{ + Msg: "eventutil.BuildEvent failed", + } + } + // check to see if this user can perform this operation + stateEvents := make([]*gomatrixserverlib.Event, len(queryRes.StateEvents)) + for i := range queryRes.StateEvents { + stateEvents[i] = queryRes.StateEvents[i].Event + } + provider := gomatrixserverlib.NewAuthEvents(stateEvents) + if err = gomatrixserverlib.Allowed(headeredEvent.Event, &provider); err != nil { + return nil, &api.PerformError{ + Code: api.PerformErrorNotAllowed, + Msg: err.Error(), // TODO: Is this error string comprehensible to the client? + } + } + + return headeredEvent, nil +} + +func createTemporaryPowerLevels(powerLevelContent *gomatrixserverlib.PowerLevelContent, userID string) (fledglingEvent, bool) { + // Work out what power level we need in order to be able to send events + // of all types into the room. + neededPowerLevel := powerLevelContent.StateDefault + for _, powerLevel := range powerLevelContent.Events { + if powerLevel > neededPowerLevel { + neededPowerLevel = powerLevel + } + } + + // Make a copy of the existing power level content. + tempPowerLevelContent := *powerLevelContent + powerLevelsOverridden := false + + // At this point, the "Users", "Events" and "Notifications" keys are all + // pointing to the map of the original PL content, so we will specifically + // override the users map with a new one and duplicate the values deeply, + // so that we can modify them without modifying the original. + tempPowerLevelContent.Users = make(map[string]int64, len(powerLevelContent.Users)) + for key, value := range powerLevelContent.Users { + tempPowerLevelContent.Users[key] = value + } + + // If the user who is upgrading the room doesn't already have sufficient + // power, then elevate their power levels. + if tempPowerLevelContent.UserLevel(userID) < neededPowerLevel { + tempPowerLevelContent.Users[userID] = neededPowerLevel + powerLevelsOverridden = true + } + + // Then return the temporary power levels event. + return fledglingEvent{ + Type: gomatrixserverlib.MRoomPowerLevels, + Content: tempPowerLevelContent, + }, powerLevelsOverridden +} + +func (r *Upgrader) sendHeaderedEvent( + ctx context.Context, + headeredEvent *gomatrixserverlib.HeaderedEvent, +) *api.PerformError { + var inputs []api.InputRoomEvent + inputs = append(inputs, api.InputRoomEvent{ + Kind: api.KindNew, + Event: headeredEvent, + Origin: r.Cfg.Matrix.ServerName, + SendAsServer: api.DoNotSendToOtherServers, + }) + if err := api.SendInputRoomEvents(ctx, r.URSAPI, inputs, false); err != nil { + return &api.PerformError{ + Msg: "api.SendInputRoomEvents failed", + } + } + + return nil +} + +func (r *Upgrader) buildEvent( + builder *gomatrixserverlib.EventBuilder, + provider gomatrixserverlib.AuthEventProvider, + evTime time.Time, + roomVersion gomatrixserverlib.RoomVersion, +) (*gomatrixserverlib.Event, error) { + eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder) + if err != nil { + return nil, err + } + refs, err := eventsNeeded.AuthEventReferences(provider) + if err != nil { + return nil, err + } + builder.AuthEvents = refs + event, err := builder.Build( + evTime, r.Cfg.Matrix.ServerName, r.Cfg.Matrix.KeyID, + r.Cfg.Matrix.PrivateKey, roomVersion, + ) + if err != nil { + return nil, fmt.Errorf("cannot build event %s : Builder failed to build. %w", builder.Type, err) + } + return event, nil +} diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index 471c6fb42..7e4d56684 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -220,7 +220,7 @@ func (r *Queryer) QueryMembershipsForRoom( if request.Sender == "" { var events []types.Event var eventNIDs []types.EventNID - eventNIDs, err = r.DB.GetMembershipEventNIDsForRoom(ctx, info.RoomNID, request.JoinedOnly, false) + eventNIDs, err = r.DB.GetMembershipEventNIDsForRoom(ctx, info.RoomNID, request.JoinedOnly, request.LocalOnly) if err != nil { return fmt.Errorf("r.DB.GetMembershipEventNIDsForRoom: %w", err) } diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go index 99c596606..d55805a91 100644 --- a/roomserver/inthttp/client.go +++ b/roomserver/inthttp/client.go @@ -32,6 +32,7 @@ const ( RoomserverPerformInvitePath = "/roomserver/performInvite" RoomserverPerformPeekPath = "/roomserver/performPeek" RoomserverPerformUnpeekPath = "/roomserver/performUnpeek" + RoomserverPerformRoomUpgradePath = "/roomserver/performRoomUpgrade" RoomserverPerformJoinPath = "/roomserver/performJoin" RoomserverPerformLeavePath = "/roomserver/performLeave" RoomserverPerformBackfillPath = "/roomserver/performBackfill" @@ -252,6 +253,23 @@ func (h *httpRoomserverInternalAPI) PerformUnpeek( } } +func (h *httpRoomserverInternalAPI) PerformRoomUpgrade( + ctx context.Context, + request *api.PerformRoomUpgradeRequest, + response *api.PerformRoomUpgradeResponse, +) { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformRoomUpgrade") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverPerformRoomUpgradePath + err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + if err != nil { + response.Error = &api.PerformError{ + Msg: fmt.Sprintf("failed to communicate with roomserver: %s", err), + } + } +} + func (h *httpRoomserverInternalAPI) PerformLeave( ctx context.Context, request *api.PerformLeaveRequest, diff --git a/roomserver/inthttp/server.go b/roomserver/inthttp/server.go index 691a45830..0b27b5a8d 100644 --- a/roomserver/inthttp/server.go +++ b/roomserver/inthttp/server.go @@ -96,6 +96,17 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + internalAPIMux.Handle(RoomserverPerformRoomUpgradePath, + httputil.MakeInternalAPI("performRoomUpgrade", func(req *http.Request) util.JSONResponse { + var request api.PerformRoomUpgradeRequest + var response api.PerformRoomUpgradeResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + r.PerformRoomUpgrade(req.Context(), &request, &response) + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) internalAPIMux.Handle(RoomserverPerformPublishPath, httputil.MakeInternalAPI("performPublish", func(req *http.Request) util.JSONResponse { var request api.PerformPublishRequest diff --git a/setup/base/base.go b/setup/base/base.go index c3ed05416..43d613b0c 100644 --- a/setup/base/base.go +++ b/setup/base/base.go @@ -45,8 +45,6 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" asinthttp "github.com/matrix-org/dendrite/appservice/inthttp" - eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" - eduinthttp "github.com/matrix-org/dendrite/eduserver/inthttp" federationAPI "github.com/matrix-org/dendrite/federationapi/api" federationIntHTTP "github.com/matrix-org/dendrite/federationapi/inthttp" keyserverAPI "github.com/matrix-org/dendrite/keyserver/api" @@ -247,15 +245,6 @@ func (b *BaseDendrite) UserAPIClient() userapi.UserInternalAPI { return userAPI } -// EDUServerClient returns EDUServerInputAPI for hitting the EDU server over HTTP -func (b *BaseDendrite) EDUServerClient() eduServerAPI.EDUServerInputAPI { - e, err := eduinthttp.NewEDUServerClient(b.Cfg.EDUServerURL(), b.apiHttpClient) - if err != nil { - logrus.WithError(err).Panic("EDUServerClient failed", b.apiHttpClient) - } - return e -} - // FederationAPIHTTPClient returns FederationInternalAPI for hitting // the federation API server over HTTP func (b *BaseDendrite) FederationAPIHTTPClient() federationAPI.FederationInternalAPI { @@ -289,6 +278,7 @@ func (b *BaseDendrite) CreateAccountsDB() userdb.Database { b.Cfg.UserAPI.BCryptCost, b.Cfg.UserAPI.OpenIDTokenLifetimeMS, userapi.DefaultLoginTokenLifetime, + b.Cfg.Global.ServerNotices.LocalPart, ) if err != nil { logrus.WithError(err).Panicf("failed to connect to accounts db") diff --git a/setup/config/config.go b/setup/config/config.go index eb371a54b..e03518e24 100644 --- a/setup/config/config.go +++ b/setup/config/config.go @@ -56,7 +56,6 @@ type Dendrite struct { Global Global `yaml:"global"` AppServiceAPI AppServiceAPI `yaml:"app_service_api"` ClientAPI ClientAPI `yaml:"client_api"` - EDUServer EDUServer `yaml:"edu_server"` FederationAPI FederationAPI `yaml:"federation_api"` KeyServer KeyServer `yaml:"key_server"` MediaAPI MediaAPI `yaml:"media_api"` @@ -296,7 +295,6 @@ func (c *Dendrite) Defaults(generate bool) { c.Global.Defaults(generate) c.ClientAPI.Defaults(generate) - c.EDUServer.Defaults(generate) c.FederationAPI.Defaults(generate) c.KeyServer.Defaults(generate) c.MediaAPI.Defaults(generate) @@ -314,8 +312,7 @@ func (c *Dendrite) Verify(configErrs *ConfigErrors, isMonolith bool) { Verify(configErrs *ConfigErrors, isMonolith bool) } for _, c := range []verifiable{ - &c.Global, &c.ClientAPI, - &c.EDUServer, &c.FederationAPI, + &c.Global, &c.ClientAPI, &c.FederationAPI, &c.KeyServer, &c.MediaAPI, &c.RoomServer, &c.SyncAPI, &c.UserAPI, &c.AppServiceAPI, &c.MSCs, @@ -327,7 +324,6 @@ func (c *Dendrite) Verify(configErrs *ConfigErrors, isMonolith bool) { func (c *Dendrite) Wiring() { c.Global.JetStream.Matrix = &c.Global c.ClientAPI.Matrix = &c.Global - c.EDUServer.Matrix = &c.Global c.FederationAPI.Matrix = &c.Global c.KeyServer.Matrix = &c.Global c.MediaAPI.Matrix = &c.Global @@ -519,15 +515,6 @@ func (config *Dendrite) UserAPIURL() string { return string(config.UserAPI.InternalAPI.Connect) } -// EDUServerURL returns an HTTP URL for where the EDU server is listening. -func (config *Dendrite) EDUServerURL() string { - // Hard code the EDU server to talk HTTP for now. - // If we support HTTPS we need to think of a practical way to do certificate validation. - // People setting up servers shouldn't need to get a certificate valid for the public - // internet for an internal API. - return string(config.EDUServer.InternalAPI.Connect) -} - // KeyServerURL returns an HTTP URL for where the key server is listening. func (config *Dendrite) KeyServerURL() string { // Hard code the key server to talk HTTP for now. diff --git a/setup/config/config_eduserver.go b/setup/config/config_eduserver.go deleted file mode 100644 index e7ed36aa0..000000000 --- a/setup/config/config_eduserver.go +++ /dev/null @@ -1,17 +0,0 @@ -package config - -type EDUServer struct { - Matrix *Global `yaml:"-"` - - InternalAPI InternalAPIOptions `yaml:"internal_api"` -} - -func (c *EDUServer) Defaults(generate bool) { - c.InternalAPI.Listen = "http://localhost:7778" - c.InternalAPI.Connect = "http://localhost:7778" -} - -func (c *EDUServer) Verify(configErrs *ConfigErrors, isMonolith bool) { - checkURL(configErrs, "edu_server.internal_api.listen", string(c.InternalAPI.Listen)) - checkURL(configErrs, "edu_server.internal_api.connect", string(c.InternalAPI.Connect)) -} diff --git a/setup/config/config_federationapi.go b/setup/config/config_federationapi.go index 95e705033..176334dd8 100644 --- a/setup/config/config_federationapi.go +++ b/setup/config/config_federationapi.go @@ -12,13 +12,6 @@ type FederationAPI struct { // send transactions to remote servers. Database DatabaseOptions `yaml:"database"` - // List of paths to X509 certificates used by the external federation listeners. - // These are used to calculate the TLS fingerprints to publish for this server. - // Other matrix servers talking to this server will expect the x509 certificate - // to match one of these certificates. - // The certificates should be in PEM format. - FederationCertificatePaths []Path `yaml:"federation_certificates"` - // Federation failure threshold. How many consecutive failures that we should // tolerate when sending federation requests to a specific server. The backoff // is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds, etc. @@ -57,8 +50,6 @@ func (c *FederationAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { checkURL(configErrs, "federation_api.external_api.listen", string(c.ExternalAPI.Listen)) } checkNotEmpty(configErrs, "federation_api.database.connection_string", string(c.Database.ConnectionString)) - // TODO: not applicable always, e.g. in demos - //checkNotZero(configErrs, "federation_api.federation_certificates", int64(len(c.FederationCertificatePaths))) } // The config for setting a proxy to use for server->server requests diff --git a/setup/config/config_global.go b/setup/config/config_global.go index b947f2076..c1650f077 100644 --- a/setup/config/config_global.go +++ b/setup/config/config_global.go @@ -41,6 +41,9 @@ type Global struct { // to other servers and the federation API will not be exposed. DisableFederation bool `yaml:"disable_federation"` + // Configures the handling of presence events. + Presence PresenceOptions `yaml:"presence"` + // List of domains that the server will trust as identity servers to // verify third-party identifiers. // Defaults to an empty array. @@ -225,3 +228,11 @@ func (c *DNSCacheOptions) Verify(configErrs *ConfigErrors, isMonolith bool) { checkPositive(configErrs, "cache_size", int64(c.CacheSize)) checkPositive(configErrs, "cache_lifetime", int64(c.CacheLifetime)) } + +// PresenceOptions defines possible configurations for presence events. +type PresenceOptions struct { + // Whether inbound presence events are allowed + EnableInbound bool `yaml:"enable_inbound"` + // Whether outbound presence events are allowed + EnableOutbound bool `yaml:"enable_outbound"` +} diff --git a/setup/config/config_test.go b/setup/config/config_test.go index e6f0a493e..cbc57ad18 100644 --- a/setup/config/config_test.go +++ b/setup/config/config_test.go @@ -101,28 +101,12 @@ current_state_server: max_open_conns: 100 max_idle_conns: 2 conn_max_lifetime: -1 -edu_server: - internal_api: - listen: http://localhost:7778 - connect: http://localhost:7778 federation_api: internal_api: listen: http://localhost:7772 connect: http://localhost:7772 external_api: listen: http://[::]:8072 - federation_certificates: [] -federation_sender: - internal_api: - listen: http://localhost:7775 - connect: http://localhost:7775 - database: - connection_string: file:federationapi.db - max_open_conns: 100 - max_idle_conns: 2 - conn_max_lifetime: -1 - send_max_retries: 16 - disable_tls_validation: false key_server: internal_api: listen: http://localhost:7779 diff --git a/setup/jetstream/nats.go b/setup/jetstream/nats.go index 328cf9155..1c8a89e8d 100644 --- a/setup/jetstream/nats.go +++ b/setup/jetstream/nats.go @@ -13,12 +13,22 @@ import ( "github.com/sirupsen/logrus" natsserver "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" natsclient "github.com/nats-io/nats.go" ) var natsServer *natsserver.Server var natsServerMutex sync.Mutex +func PrepareForTests() (*process.ProcessContext, nats.JetStreamContext, *nats.Conn) { + cfg := &config.Dendrite{} + cfg.Defaults(true) + cfg.Global.JetStream.InMemory = true + pc := process.NewProcessContext() + js, jc := Prepare(pc, &cfg.Global.JetStream) + return pc, js, jc +} + func Prepare(process *process.ProcessContext, cfg *config.JetStream) (natsclient.JetStreamContext, *natsclient.Conn) { // check if we need an in-process NATS Server if len(cfg.Addresses) != 0 { @@ -157,5 +167,26 @@ func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsc } } + // Clean up old consumers so that interest-based consumers do the + // right thing. + for stream, consumers := range map[string][]string{ + OutputClientData: {"SyncAPIClientAPIConsumer"}, + OutputReceiptEvent: {"SyncAPIEDUServerReceiptConsumer", "FederationAPIEDUServerConsumer"}, + OutputSendToDeviceEvent: {"SyncAPIEDUServerSendToDeviceConsumer", "FederationAPIEDUServerConsumer"}, + OutputTypingEvent: {"SyncAPIEDUServerTypingConsumer", "FederationAPIEDUServerConsumer"}, + } { + streamName := cfg.Matrix.JetStream.Prefixed(stream) + for _, consumer := range consumers { + consumerName := cfg.Matrix.JetStream.Prefixed(consumer) + "Pull" + consumerInfo, err := s.ConsumerInfo(streamName, consumerName) + if err != nil || consumerInfo == nil { + continue + } + if err = s.DeleteConsumer(streamName, consumerName); err != nil { + logrus.WithError(err).Errorf("Unable to clean up old consumer %q for stream %q", consumer, stream) + } + } + } + return s, nc } diff --git a/setup/jetstream/streams.go b/setup/jetstream/streams.go index aa979924b..6594e6941 100644 --- a/setup/jetstream/streams.go +++ b/setup/jetstream/streams.go @@ -9,8 +9,9 @@ import ( ) const ( - UserID = "user_id" - RoomID = "room_id" + UserID = "user_id" + RoomID = "room_id" + EventID = "event_id" ) var ( @@ -24,6 +25,8 @@ var ( OutputReceiptEvent = "OutputReceiptEvent" OutputStreamEvent = "OutputStreamEvent" OutputReadUpdate = "OutputReadUpdate" + RequestPresence = "GetPresence" + OutputPresenceEvent = "OutputPresenceEvent" ) var safeCharacters = regexp.MustCompile("[^A-Za-z0-9$]+") @@ -88,4 +91,10 @@ var streams = []*nats.StreamConfig{ Retention: nats.InterestPolicy, Storage: nats.FileStorage, }, + { + Name: OutputPresenceEvent, + Retention: nats.InterestPolicy, + Storage: nats.MemoryStorage, + MaxAge: time.Minute * 5, + }, } diff --git a/setup/monolith.go b/setup/monolith.go index 8e6240d37..32f1a6494 100644 --- a/setup/monolith.go +++ b/setup/monolith.go @@ -19,7 +19,6 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi" "github.com/matrix-org/dendrite/clientapi/api" - eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/federationapi" federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/transactions" @@ -43,30 +42,34 @@ type Monolith struct { Client *gomatrixserverlib.Client FedClient *gomatrixserverlib.FederationClient - AppserviceAPI appserviceAPI.AppServiceQueryAPI - EDUInternalAPI eduServerAPI.EDUServerInputAPI - FederationAPI federationAPI.FederationInternalAPI - RoomserverAPI roomserverAPI.RoomserverInternalAPI - UserAPI userapi.UserInternalAPI - KeyAPI keyAPI.KeyInternalAPI + AppserviceAPI appserviceAPI.AppServiceQueryAPI + FederationAPI federationAPI.FederationInternalAPI + RoomserverAPI roomserverAPI.RoomserverInternalAPI + UserAPI userapi.UserInternalAPI + KeyAPI keyAPI.KeyInternalAPI // Optional - ExtPublicRoomsProvider api.ExtraPublicRoomsProvider + ExtPublicRoomsProvider api.ExtraPublicRoomsProvider + ExtUserDirectoryProvider userapi.UserDirectoryProvider } // AddAllPublicRoutes attaches all public paths to the given router func (m *Monolith) AddAllPublicRoutes(process *process.ProcessContext, csMux, ssMux, keyMux, wkMux, mediaMux, synapseMux *mux.Router) { + userDirectoryProvider := m.ExtUserDirectoryProvider + if userDirectoryProvider == nil { + userDirectoryProvider = m.UserAPI + } clientapi.AddPublicRoutes( process, csMux, synapseMux, &m.Config.ClientAPI, m.FedClient, m.RoomserverAPI, - m.EDUInternalAPI, m.AppserviceAPI, transactions.New(), - m.FederationAPI, m.UserAPI, m.KeyAPI, + m.AppserviceAPI, transactions.New(), + m.FederationAPI, m.UserAPI, userDirectoryProvider, m.KeyAPI, m.ExtPublicRoomsProvider, &m.Config.MSCs, ) federationapi.AddPublicRoutes( - ssMux, keyMux, wkMux, &m.Config.FederationAPI, m.UserAPI, m.FedClient, + process, ssMux, keyMux, wkMux, &m.Config.FederationAPI, m.UserAPI, m.FedClient, m.KeyRing, m.RoomserverAPI, m.FederationAPI, - m.EDUInternalAPI, m.KeyAPI, &m.Config.MSCs, nil, + m.KeyAPI, &m.Config.MSCs, nil, ) mediaapi.AddPublicRoutes(mediaMux, &m.Config.MediaAPI, &m.Config.ClientAPI.RateLimiting, m.UserAPI, m.Client) syncapi.AddPublicRoutes( diff --git a/show-expected-fail-tests.sh b/show-expected-fail-tests.sh index 320d4ebd3..3ed937a0f 100755 --- a/show-expected-fail-tests.sh +++ b/show-expected-fail-tests.sh @@ -89,17 +89,17 @@ if [ -n "${tests_to_add}" ] && [ -n "${already_in_whitelist}" ]; then fi if [ -n "${tests_to_add}" ]; then - echo "**ERROR**: The following tests passed but are not present in \`$2\`. Please append them to the file:" - echo "\`\`\`" - echo -e "${tests_to_add}" - echo "\`\`\`" + echo "::error::The following tests passed but are not present in \`$2\`. Please append them to the file:" + echo "::group::Passing tests" + echo -e "${tests_to_add}" + echo "::endgroup::" fi if [ -n "${already_in_whitelist}" ]; then - echo "**WARN**: Tests in the whitelist still marked as **expected fail**:" - echo "\`\`\`" - echo -e "${already_in_whitelist}" - echo "\`\`\`" + echo "::warning::Tests in the whitelist still marked as **expected fail**:" + echo "::group::Still marked as expected fail" + echo -e "${already_in_whitelist}" + echo "::endgroup::" fi exit ${fail_build} diff --git a/syncapi/consumers/clientapi.go b/syncapi/consumers/clientapi.go index 40c1cd3d6..eec369c1a 100644 --- a/syncapi/consumers/clientapi.go +++ b/syncapi/consumers/clientapi.go @@ -62,7 +62,7 @@ func NewOutputClientDataConsumer( ctx: process.Context(), jetstream: js, topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputClientData), - durable: cfg.Matrix.JetStream.Durable("SyncAPIClientAPIConsumer"), + durable: cfg.Matrix.JetStream.Durable("SyncAPIAccountDataConsumer"), db: store, notifier: notifier, stream: stream, @@ -119,6 +119,15 @@ func (s *OutputClientDataConsumer) onMessage(ctx context.Context, msg *nats.Msg) return false } + if output.IgnoredUsers != nil { + if err := s.db.UpdateIgnoresForUser(ctx, userID, output.IgnoredUsers); err != nil { + log.WithError(err).WithFields(logrus.Fields{ + "user_id": userID, + }).Errorf("Failed to update ignored users") + sentry.CaptureException(err) + } + } + s.stream.Advance(streamPos) s.notifier.OnNewAccountData(userID, types.StreamingToken{AccountDataPosition: streamPos}) diff --git a/syncapi/consumers/presence.go b/syncapi/consumers/presence.go new file mode 100644 index 000000000..b198b2292 --- /dev/null +++ b/syncapi/consumers/presence.go @@ -0,0 +1,158 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package consumers + +import ( + "context" + "strconv" + + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" + "github.com/matrix-org/dendrite/syncapi/notifier" + "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/nats-io/nats.go" + "github.com/sirupsen/logrus" +) + +// OutputTypingEventConsumer consumes events that originated in the EDU server. +type PresenceConsumer struct { + ctx context.Context + jetstream nats.JetStreamContext + nats *nats.Conn + durable string + requestTopic string + presenceTopic string + db storage.Database + stream types.StreamProvider + notifier *notifier.Notifier + deviceAPI api.UserDeviceAPI + cfg *config.SyncAPI +} + +// NewPresenceConsumer creates a new PresenceConsumer. +// Call Start() to begin consuming events. +func NewPresenceConsumer( + process *process.ProcessContext, + cfg *config.SyncAPI, + js nats.JetStreamContext, + nats *nats.Conn, + db storage.Database, + notifier *notifier.Notifier, + stream types.StreamProvider, + deviceAPI api.UserDeviceAPI, +) *PresenceConsumer { + return &PresenceConsumer{ + ctx: process.Context(), + nats: nats, + jetstream: js, + durable: cfg.Matrix.JetStream.Durable("SyncAPIPresenceConsumer"), + presenceTopic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent), + requestTopic: cfg.Matrix.JetStream.Prefixed(jetstream.RequestPresence), + db: db, + notifier: notifier, + stream: stream, + deviceAPI: deviceAPI, + cfg: cfg, + } +} + +// Start consuming typing events. +func (s *PresenceConsumer) Start() error { + // Normal NATS subscription, used by Request/Reply + _, err := s.nats.Subscribe(s.requestTopic, func(msg *nats.Msg) { + userID := msg.Header.Get(jetstream.UserID) + presence, err := s.db.GetPresence(context.Background(), userID) + m := &nats.Msg{ + Header: nats.Header{}, + } + if err != nil { + m.Header.Set("error", err.Error()) + if err = msg.RespondMsg(m); err != nil { + logrus.WithError(err).Error("Unable to respond to messages") + } + return + } + + deviceRes := api.QueryDevicesResponse{} + if err = s.deviceAPI.QueryDevices(s.ctx, &api.QueryDevicesRequest{UserID: userID}, &deviceRes); err != nil { + m.Header.Set("error", err.Error()) + if err = msg.RespondMsg(m); err != nil { + logrus.WithError(err).Error("Unable to respond to messages") + } + return + } + + for i := range deviceRes.Devices { + if int64(presence.LastActiveTS) < deviceRes.Devices[i].LastSeenTS { + presence.LastActiveTS = gomatrixserverlib.Timestamp(deviceRes.Devices[i].LastSeenTS) + } + } + + m.Header.Set(jetstream.UserID, presence.UserID) + m.Header.Set("presence", presence.ClientFields.Presence) + m.Header.Set("status_msg", *presence.ClientFields.StatusMsg) + m.Header.Set("last_active_ts", strconv.Itoa(int(presence.LastActiveTS))) + + if err = msg.RespondMsg(m); err != nil { + logrus.WithError(err).Error("Unable to respond to messages") + return + } + }) + if err != nil { + return err + } + if !s.cfg.Matrix.Presence.EnableInbound && !s.cfg.Matrix.Presence.EnableOutbound { + return nil + } + return jetstream.JetStreamConsumer( + s.ctx, s.jetstream, s.presenceTopic, s.durable, s.onMessage, + nats.DeliverAll(), nats.ManualAck(), nats.HeadersOnly(), + ) +} + +func (s *PresenceConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool { + userID := msg.Header.Get(jetstream.UserID) + presence := msg.Header.Get("presence") + timestamp := msg.Header.Get("last_active_ts") + fromSync, _ := strconv.ParseBool(msg.Header.Get("from_sync")) + + logrus.Debugf("syncAPI received presence event: %+v", msg.Header) + + ts, err := strconv.Atoi(timestamp) + if err != nil { + return true + } + + var statusMsg *string = nil + if data, ok := msg.Header["status_msg"]; ok && len(data) > 0 { + newMsg := msg.Header.Get("status_msg") + statusMsg = &newMsg + } + // OK is already checked, so no need to do it again + p, _ := types.PresenceFromString(presence) + pos, err := s.db.UpdatePresence(ctx, userID, p, statusMsg, gomatrixserverlib.Timestamp(ts), fromSync) + if err != nil { + return true + } + + s.stream.Advance(pos) + s.notifier.OnNewPresence(types.StreamingToken{PresencePosition: pos}, userID) + + return true +} diff --git a/syncapi/consumers/eduserver_receipts.go b/syncapi/consumers/receipts.go similarity index 87% rename from syncapi/consumers/eduserver_receipts.go rename to syncapi/consumers/receipts.go index ab79998ea..6bb0747f0 100644 --- a/syncapi/consumers/eduserver_receipts.go +++ b/syncapi/consumers/receipts.go @@ -17,11 +17,10 @@ package consumers import ( "context" "database/sql" - "encoding/json" "fmt" + "strconv" "github.com/getsentry/sentry-go" - "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" @@ -63,7 +62,7 @@ func NewOutputReceiptEventConsumer( ctx: process.Context(), jetstream: js, topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent), - durable: cfg.Matrix.JetStream.Durable("SyncAPIEDUServerReceiptConsumer"), + durable: cfg.Matrix.JetStream.Durable("SyncAPIReceiptConsumer"), db: store, notifier: notifier, stream: stream, @@ -72,7 +71,7 @@ func NewOutputReceiptEventConsumer( } } -// Start consuming from EDU api +// Start consuming receipts events. func (s *OutputReceiptEventConsumer) Start() error { return jetstream.JetStreamConsumer( s.ctx, s.jetstream, s.topic, s.durable, s.onMessage, @@ -81,14 +80,23 @@ func (s *OutputReceiptEventConsumer) Start() error { } func (s *OutputReceiptEventConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool { - var output api.OutputReceiptEvent - if err := json.Unmarshal(msg.Data, &output); err != nil { + output := types.OutputReceiptEvent{ + UserID: msg.Header.Get(jetstream.UserID), + RoomID: msg.Header.Get(jetstream.RoomID), + EventID: msg.Header.Get(jetstream.EventID), + Type: msg.Header.Get("type"), + } + + timestamp, err := strconv.Atoi(msg.Header.Get("timestamp")) + if err != nil { // If the message was invalid, log it and move on to the next message in the stream - log.WithError(err).Errorf("EDU server output log: message parse failure") + log.WithError(err).Errorf("output log: message parse failure") sentry.CaptureException(err) return true } + output.Timestamp = gomatrixserverlib.Timestamp(timestamp) + streamPos, err := s.db.StoreReceipt( s.ctx, output.RoomID, @@ -117,7 +125,7 @@ func (s *OutputReceiptEventConsumer) onMessage(ctx context.Context, msg *nats.Ms return true } -func (s *OutputReceiptEventConsumer) sendReadUpdate(ctx context.Context, output api.OutputReceiptEvent) error { +func (s *OutputReceiptEventConsumer) sendReadUpdate(ctx context.Context, output types.OutputReceiptEvent) error { if output.Type != "m.read" { return nil } diff --git a/syncapi/consumers/eduserver_sendtodevice.go b/syncapi/consumers/sendtodevice.go similarity index 87% rename from syncapi/consumers/eduserver_sendtodevice.go rename to syncapi/consumers/sendtodevice.go index bdbe77352..0b9153fcd 100644 --- a/syncapi/consumers/eduserver_sendtodevice.go +++ b/syncapi/consumers/sendtodevice.go @@ -19,7 +19,6 @@ import ( "encoding/json" "github.com/getsentry/sentry-go" - "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" @@ -58,7 +57,7 @@ func NewOutputSendToDeviceEventConsumer( ctx: process.Context(), jetstream: js, topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), - durable: cfg.Matrix.JetStream.Durable("SyncAPIEDUServerSendToDeviceConsumer"), + durable: cfg.Matrix.JetStream.Durable("SyncAPISendToDeviceConsumer"), db: store, serverName: cfg.Matrix.ServerName, notifier: notifier, @@ -66,7 +65,7 @@ func NewOutputSendToDeviceEventConsumer( } } -// Start consuming from EDU api +// Start consuming send-to-device events. func (s *OutputSendToDeviceEventConsumer) Start() error { return jetstream.JetStreamConsumer( s.ctx, s.jetstream, s.topic, s.durable, s.onMessage, @@ -75,15 +74,8 @@ func (s *OutputSendToDeviceEventConsumer) Start() error { } func (s *OutputSendToDeviceEventConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool { - var output api.OutputSendToDeviceEvent - if err := json.Unmarshal(msg.Data, &output); err != nil { - // If the message was invalid, log it and move on to the next message in the stream - log.WithError(err).Errorf("EDU server output log: message parse failure") - sentry.CaptureException(err) - return true - } - - _, domain, err := gomatrixserverlib.SplitID('@', output.UserID) + userID := msg.Header.Get(jetstream.UserID) + _, domain, err := gomatrixserverlib.SplitID('@', userID) if err != nil { sentry.CaptureException(err) return true @@ -92,12 +84,20 @@ func (s *OutputSendToDeviceEventConsumer) onMessage(ctx context.Context, msg *na return true } + var output types.OutputSendToDeviceEvent + if err = json.Unmarshal(msg.Data, &output); err != nil { + // If the message was invalid, log it and move on to the next message in the stream + log.WithError(err).Errorf("output log: message parse failure") + sentry.CaptureException(err) + return true + } + util.GetLogger(context.TODO()).WithFields(log.Fields{ "sender": output.Sender, "user_id": output.UserID, "device_id": output.DeviceID, "event_type": output.Type, - }).Info("sync API received send-to-device event from EDU server") + }).Debugf("sync API received send-to-device event from the clientapi/federationsender") streamPos, err := s.db.StoreNewSendForDeviceMessage( s.ctx, output.UserID, output.DeviceID, output.SendToDeviceEvent, diff --git a/syncapi/consumers/eduserver_typing.go b/syncapi/consumers/typing.go similarity index 67% rename from syncapi/consumers/eduserver_typing.go rename to syncapi/consumers/typing.go index c2828c7fc..48e484ec5 100644 --- a/syncapi/consumers/eduserver_typing.go +++ b/syncapi/consumers/typing.go @@ -16,16 +16,14 @@ package consumers import ( "context" - "encoding/json" + "strconv" + "time" - "github.com/getsentry/sentry-go" - "github.com/matrix-org/dendrite/eduserver/api" - "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/syncapi/notifier" - "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" "github.com/nats-io/nats.go" log "github.com/sirupsen/logrus" @@ -37,7 +35,7 @@ type OutputTypingEventConsumer struct { jetstream nats.JetStreamContext durable string topic string - eduCache *cache.EDUCache + eduCache *caching.EDUCache stream types.StreamProvider notifier *notifier.Notifier } @@ -48,8 +46,7 @@ func NewOutputTypingEventConsumer( process *process.ProcessContext, cfg *config.SyncAPI, js nats.JetStreamContext, - store storage.Database, - eduCache *cache.EDUCache, + eduCache *caching.EDUCache, notifier *notifier.Notifier, stream types.StreamProvider, ) *OutputTypingEventConsumer { @@ -57,14 +54,14 @@ func NewOutputTypingEventConsumer( ctx: process.Context(), jetstream: js, topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent), - durable: cfg.Matrix.JetStream.Durable("SyncAPIEDUServerTypingConsumer"), + durable: cfg.Matrix.JetStream.Durable("SyncAPITypingConsumer"), eduCache: eduCache, notifier: notifier, stream: stream, } } -// Start consuming from EDU api +// Start consuming typing events. func (s *OutputTypingEventConsumer) Start() error { return jetstream.JetStreamConsumer( s.ctx, s.jetstream, s.topic, s.durable, s.onMessage, @@ -73,34 +70,40 @@ func (s *OutputTypingEventConsumer) Start() error { } func (s *OutputTypingEventConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool { - var output api.OutputTypingEvent - if err := json.Unmarshal(msg.Data, &output); err != nil { - // If the message was invalid, log it and move on to the next message in the stream - log.WithError(err).Errorf("EDU server output log: message parse failure") - sentry.CaptureException(err) + roomID := msg.Header.Get(jetstream.RoomID) + userID := msg.Header.Get(jetstream.UserID) + typing, err := strconv.ParseBool(msg.Header.Get("typing")) + if err != nil { + log.WithError(err).Errorf("output log: typing parse failure") + return true + } + timeout, err := strconv.Atoi(msg.Header.Get("timeout_ms")) + if err != nil { + log.WithError(err).Errorf("output log: timeout_ms parse failure") return true } log.WithFields(log.Fields{ - "room_id": output.Event.RoomID, - "user_id": output.Event.UserID, - "typing": output.Event.Typing, - }).Debug("received data from EDU server") + "room_id": roomID, + "user_id": userID, + "typing": typing, + "timeout": timeout, + }).Debug("syncapi received EDU data from client api") var typingPos types.StreamPosition - typingEvent := output.Event - if typingEvent.Typing { + if typing { + expiry := time.Now().Add(time.Duration(timeout) * time.Millisecond) typingPos = types.StreamPosition( - s.eduCache.AddTypingUser(typingEvent.UserID, typingEvent.RoomID, output.ExpireTime), + s.eduCache.AddTypingUser(userID, roomID, &expiry), ) } else { typingPos = types.StreamPosition( - s.eduCache.RemoveUser(typingEvent.UserID, typingEvent.RoomID), + s.eduCache.RemoveUser(userID, roomID), ) } s.stream.Advance(typingPos) - s.notifier.OnNewTyping(output.Event.RoomID, types.StreamingToken{TypingPosition: typingPos}) + s.notifier.OnNewTyping(roomID, types.StreamingToken{TypingPosition: typingPos}) return true } diff --git a/syncapi/notifier/notifier.go b/syncapi/notifier/notifier.go index 6a641e6f8..443744b6f 100644 --- a/syncapi/notifier/notifier.go +++ b/syncapi/notifier/notifier.go @@ -25,40 +25,53 @@ import ( log "github.com/sirupsen/logrus" ) +// NOTE: ALL FUNCTIONS IN THIS FILE PREFIXED WITH _ ARE NOT THREAD-SAFE +// AND MUST ONLY BE CALLED WHEN THE NOTIFIER LOCK IS HELD! + // Notifier will wake up sleeping requests when there is some new data. // It does not tell requests what that data is, only the sync position which // they can use to get at it. This is done to prevent races whereby we tell the caller // the event, but the token has already advanced by the time they fetch it, resulting // in missed events. type Notifier struct { + lock *sync.RWMutex // A map of RoomID => Set : Must only be accessed by the OnNewEvent goroutine roomIDToJoinedUsers map[string]userIDSet // A map of RoomID => Set : Must only be accessed by the OnNewEvent goroutine roomIDToPeekingDevices map[string]peekingDeviceSet - // Protects currPos and userStreams. - streamLock *sync.Mutex // The latest sync position currPos types.StreamingToken // A map of user_id => device_id => UserStream which can be used to wake a given user's /sync request. userDeviceStreams map[string]map[string]*UserDeviceStream // The last time we cleaned out stale entries from the userStreams map lastCleanUpTime time.Time + // This map is reused to prevent allocations and GC pressure in SharedUsers. + _sharedUserMap map[string]struct{} } // NewNotifier creates a new notifier set to the given sync position. // In order for this to be of any use, the Notifier needs to be told all rooms and // the joined users within each of them by calling Notifier.Load(*storage.SyncServerDatabase). -func NewNotifier(currPos types.StreamingToken) *Notifier { +func NewNotifier() *Notifier { return &Notifier{ - currPos: currPos, roomIDToJoinedUsers: make(map[string]userIDSet), roomIDToPeekingDevices: make(map[string]peekingDeviceSet), userDeviceStreams: make(map[string]map[string]*UserDeviceStream), - streamLock: &sync.Mutex{}, + lock: &sync.RWMutex{}, lastCleanUpTime: time.Now(), + _sharedUserMap: map[string]struct{}{}, } } +// SetCurrentPosition sets the current streaming positions. +// This must be called directly after NewNotifier and initialising the streams. +func (n *Notifier) SetCurrentPosition(currPos types.StreamingToken) { + n.lock.Lock() + defer n.lock.Unlock() + + n.currPos = currPos +} + // OnNewEvent is called when a new event is received from the room server. Must only be // called from a single goroutine, to avoid races between updates which could set the // current sync position incorrectly. @@ -75,17 +88,16 @@ func (n *Notifier) OnNewEvent( ) { // update the current position then notify relevant /sync streams. // This needs to be done PRIOR to waking up users as they will read this value. - n.streamLock.Lock() - defer n.streamLock.Unlock() - + n.lock.Lock() + defer n.lock.Unlock() n.currPos.ApplyUpdates(posUpdate) - n.removeEmptyUserStreams() + n._removeEmptyUserStreams() if ev != nil { // Map this event's room_id to a list of joined users, and wake them up. - usersToNotify := n.joinedUsers(ev.RoomID()) + usersToNotify := n._joinedUsers(ev.RoomID()) // Map this event's room_id to a list of peeking devices, and wake them up. - peekingDevicesToNotify := n.PeekingDevices(ev.RoomID()) + peekingDevicesToNotify := n._peekingDevices(ev.RoomID()) // If this is an invite, also add in the invitee to this list. if ev.Type() == "m.room.member" && ev.StateKey() != nil { targetUserID := *ev.StateKey() @@ -103,20 +115,20 @@ func (n *Notifier) OnNewEvent( // Manually append the new user's ID so they get notified // along all members in the room usersToNotify = append(usersToNotify, targetUserID) - n.addJoinedUser(ev.RoomID(), targetUserID) + n._addJoinedUser(ev.RoomID(), targetUserID) case gomatrixserverlib.Leave: fallthrough case gomatrixserverlib.Ban: - n.removeJoinedUser(ev.RoomID(), targetUserID) + n._removeJoinedUser(ev.RoomID(), targetUserID) } } } - n.wakeupUsers(usersToNotify, peekingDevicesToNotify, n.currPos) + n._wakeupUsers(usersToNotify, peekingDevicesToNotify, n.currPos) } else if roomID != "" { - n.wakeupUsers(n.joinedUsers(roomID), n.PeekingDevices(roomID), n.currPos) + n._wakeupUsers(n._joinedUsers(roomID), n._peekingDevices(roomID), n.currPos) } else if len(userIDs) > 0 { - n.wakeupUsers(userIDs, nil, n.currPos) + n._wakeupUsers(userIDs, nil, n.currPos) } else { log.WithFields(log.Fields{ "posUpdate": posUpdate.String, @@ -127,22 +139,22 @@ func (n *Notifier) OnNewEvent( func (n *Notifier) OnNewAccountData( userID string, posUpdate types.StreamingToken, ) { - n.streamLock.Lock() - defer n.streamLock.Unlock() + n.lock.Lock() + defer n.lock.Unlock() n.currPos.ApplyUpdates(posUpdate) - n.wakeupUsers([]string{userID}, nil, posUpdate) + n._wakeupUsers([]string{userID}, nil, posUpdate) } func (n *Notifier) OnNewPeek( roomID, userID, deviceID string, posUpdate types.StreamingToken, ) { - n.streamLock.Lock() - defer n.streamLock.Unlock() + n.lock.Lock() + defer n.lock.Unlock() n.currPos.ApplyUpdates(posUpdate) - n.addPeekingDevice(roomID, userID, deviceID) + n._addPeekingDevice(roomID, userID, deviceID) // we don't wake up devices here given the roomserver consumer will do this shortly afterwards // by calling OnNewEvent. @@ -152,11 +164,11 @@ func (n *Notifier) OnRetirePeek( roomID, userID, deviceID string, posUpdate types.StreamingToken, ) { - n.streamLock.Lock() - defer n.streamLock.Unlock() + n.lock.Lock() + defer n.lock.Unlock() n.currPos.ApplyUpdates(posUpdate) - n.removePeekingDevice(roomID, userID, deviceID) + n._removePeekingDevice(roomID, userID, deviceID) // we don't wake up devices here given the roomserver consumer will do this shortly afterwards // by calling OnRetireEvent. @@ -166,11 +178,11 @@ func (n *Notifier) OnNewSendToDevice( userID string, deviceIDs []string, posUpdate types.StreamingToken, ) { - n.streamLock.Lock() - defer n.streamLock.Unlock() + n.lock.Lock() + defer n.lock.Unlock() n.currPos.ApplyUpdates(posUpdate) - n.wakeupUserDevice(userID, deviceIDs, n.currPos) + n._wakeupUserDevice(userID, deviceIDs, n.currPos) } // OnNewReceipt updates the current position @@ -178,11 +190,11 @@ func (n *Notifier) OnNewTyping( roomID string, posUpdate types.StreamingToken, ) { - n.streamLock.Lock() - defer n.streamLock.Unlock() + n.lock.Lock() + defer n.lock.Unlock() n.currPos.ApplyUpdates(posUpdate) - n.wakeupUsers(n.joinedUsers(roomID), nil, n.currPos) + n._wakeupUsers(n._joinedUsers(roomID), nil, n.currPos) } // OnNewReceipt updates the current position @@ -190,42 +202,93 @@ func (n *Notifier) OnNewReceipt( roomID string, posUpdate types.StreamingToken, ) { - n.streamLock.Lock() - defer n.streamLock.Unlock() + n.lock.Lock() + defer n.lock.Unlock() n.currPos.ApplyUpdates(posUpdate) - n.wakeupUsers(n.joinedUsers(roomID), nil, n.currPos) + n._wakeupUsers(n._joinedUsers(roomID), nil, n.currPos) } func (n *Notifier) OnNewKeyChange( posUpdate types.StreamingToken, wakeUserID, keyChangeUserID string, ) { - n.streamLock.Lock() - defer n.streamLock.Unlock() + n.lock.Lock() + defer n.lock.Unlock() n.currPos.ApplyUpdates(posUpdate) - n.wakeupUsers([]string{wakeUserID}, nil, n.currPos) + n._wakeupUsers([]string{wakeUserID}, nil, n.currPos) } func (n *Notifier) OnNewInvite( posUpdate types.StreamingToken, wakeUserID string, ) { - n.streamLock.Lock() - defer n.streamLock.Unlock() + n.lock.Lock() + defer n.lock.Unlock() n.currPos.ApplyUpdates(posUpdate) - n.wakeupUsers([]string{wakeUserID}, nil, n.currPos) + n._wakeupUsers([]string{wakeUserID}, nil, n.currPos) } func (n *Notifier) OnNewNotificationData( userID string, posUpdate types.StreamingToken, ) { - n.streamLock.Lock() - defer n.streamLock.Unlock() + n.lock.Lock() + defer n.lock.Unlock() n.currPos.ApplyUpdates(posUpdate) - n.wakeupUsers([]string{userID}, nil, n.currPos) + n._wakeupUsers([]string{userID}, nil, n.currPos) +} + +func (n *Notifier) OnNewPresence( + posUpdate types.StreamingToken, userID string, +) { + n.lock.Lock() + defer n.lock.Unlock() + + n.currPos.ApplyUpdates(posUpdate) + sharedUsers := n._sharedUsers(userID) + sharedUsers = append(sharedUsers, userID) + + n._wakeupUsers(sharedUsers, nil, n.currPos) +} + +func (n *Notifier) SharedUsers(userID string) []string { + n.lock.RLock() + defer n.lock.RUnlock() + return n._sharedUsers(userID) +} + +func (n *Notifier) _sharedUsers(userID string) []string { + n._sharedUserMap[userID] = struct{}{} + for roomID, users := range n.roomIDToJoinedUsers { + if _, ok := users[userID]; !ok { + continue + } + for _, userID := range n._joinedUsers(roomID) { + n._sharedUserMap[userID] = struct{}{} + } + } + sharedUsers := make([]string, 0, len(n._sharedUserMap)+1) + for userID := range n._sharedUserMap { + sharedUsers = append(sharedUsers, userID) + delete(n._sharedUserMap, userID) + } + return sharedUsers +} + +func (n *Notifier) IsSharedUser(userA, userB string) bool { + n.lock.RLock() + defer n.lock.RUnlock() + var okA, okB bool + for _, users := range n.roomIDToJoinedUsers { + _, okA = users[userA] + _, okB = users[userB] + if okA && okB { + return true + } + } + return false } // GetListener returns a UserStreamListener that can be used to wait for @@ -240,16 +303,18 @@ func (n *Notifier) GetListener(req types.SyncRequest) UserDeviceStreamListener { // TODO: v1 /events 'peeking' has an 'explicit room ID' which is also tracked, // but given we don't do /events, let's pretend it doesn't exist. - n.streamLock.Lock() - defer n.streamLock.Unlock() + n.lock.Lock() + defer n.lock.Unlock() - n.removeEmptyUserStreams() + n._removeEmptyUserStreams() - return n.fetchUserDeviceStream(req.Device.UserID, req.Device.ID, true).GetListener(req.Context) + return n._fetchUserDeviceStream(req.Device.UserID, req.Device.ID, true).GetListener(req.Context) } // Load the membership states required to notify users correctly. func (n *Notifier) Load(ctx context.Context, db storage.Database) error { + n.lock.Lock() + defer n.lock.Unlock() roomToUsers, err := db.AllJoinedUsersInRooms(ctx) if err != nil { return err @@ -267,8 +332,8 @@ func (n *Notifier) Load(ctx context.Context, db storage.Database) error { // CurrentPosition returns the current sync position func (n *Notifier) CurrentPosition() types.StreamingToken { - n.streamLock.Lock() - defer n.streamLock.Unlock() + n.lock.RLock() + defer n.lock.RUnlock() return n.currPos } @@ -280,7 +345,7 @@ func (n *Notifier) setUsersJoinedToRooms(roomIDToUserIDs map[string][]string) { // This is just the bulk form of addJoinedUser for roomID, userIDs := range roomIDToUserIDs { if _, ok := n.roomIDToJoinedUsers[roomID]; !ok { - n.roomIDToJoinedUsers[roomID] = make(userIDSet) + n.roomIDToJoinedUsers[roomID] = make(userIDSet, len(userIDs)) } for _, userID := range userIDs { n.roomIDToJoinedUsers[roomID].add(userID) @@ -295,7 +360,7 @@ func (n *Notifier) setPeekingDevices(roomIDToPeekingDevices map[string][]types.P // This is just the bulk form of addPeekingDevice for roomID, peekingDevices := range roomIDToPeekingDevices { if _, ok := n.roomIDToPeekingDevices[roomID]; !ok { - n.roomIDToPeekingDevices[roomID] = make(peekingDeviceSet) + n.roomIDToPeekingDevices[roomID] = make(peekingDeviceSet, len(peekingDevices)) } for _, peekingDevice := range peekingDevices { n.roomIDToPeekingDevices[roomID].add(peekingDevice) @@ -303,11 +368,11 @@ func (n *Notifier) setPeekingDevices(roomIDToPeekingDevices map[string][]types.P } } -// wakeupUsers will wake up the sync strems for all of the devices for all of the +// _wakeupUsers will wake up the sync strems for all of the devices for all of the // specified user IDs, and also the specified peekingDevices -func (n *Notifier) wakeupUsers(userIDs []string, peekingDevices []types.PeekingDevice, newPos types.StreamingToken) { +func (n *Notifier) _wakeupUsers(userIDs []string, peekingDevices []types.PeekingDevice, newPos types.StreamingToken) { for _, userID := range userIDs { - for _, stream := range n.fetchUserStreams(userID) { + for _, stream := range n._fetchUserStreams(userID) { if stream == nil { continue } @@ -317,28 +382,27 @@ func (n *Notifier) wakeupUsers(userIDs []string, peekingDevices []types.PeekingD for _, peekingDevice := range peekingDevices { // TODO: don't bother waking up for devices whose users we already woke up - if stream := n.fetchUserDeviceStream(peekingDevice.UserID, peekingDevice.DeviceID, false); stream != nil { + if stream := n._fetchUserDeviceStream(peekingDevice.UserID, peekingDevice.DeviceID, false); stream != nil { stream.Broadcast(newPos) // wake up all goroutines Wait()ing on this stream } } } -// wakeupUserDevice will wake up the sync stream for a specific user device. Other +// _wakeupUserDevice will wake up the sync stream for a specific user device. Other // device streams will be left alone. // nolint:unused -func (n *Notifier) wakeupUserDevice(userID string, deviceIDs []string, newPos types.StreamingToken) { +func (n *Notifier) _wakeupUserDevice(userID string, deviceIDs []string, newPos types.StreamingToken) { for _, deviceID := range deviceIDs { - if stream := n.fetchUserDeviceStream(userID, deviceID, false); stream != nil { + if stream := n._fetchUserDeviceStream(userID, deviceID, false); stream != nil { stream.Broadcast(newPos) // wake up all goroutines Wait()ing on this stream } } } -// fetchUserDeviceStream retrieves a stream unique to the given device. If makeIfNotExists is true, +// _fetchUserDeviceStream retrieves a stream unique to the given device. If makeIfNotExists is true, // a stream will be made for this device if one doesn't exist and it will be returned. This // function does not wait for data to be available on the stream. -// NB: Callers should have locked the mutex before calling this function. -func (n *Notifier) fetchUserDeviceStream(userID, deviceID string, makeIfNotExists bool) *UserDeviceStream { +func (n *Notifier) _fetchUserDeviceStream(userID, deviceID string, makeIfNotExists bool) *UserDeviceStream { _, ok := n.userDeviceStreams[userID] if !ok { if !makeIfNotExists { @@ -359,57 +423,56 @@ func (n *Notifier) fetchUserDeviceStream(userID, deviceID string, makeIfNotExist return stream } -// fetchUserStreams retrieves all streams for the given user. If makeIfNotExists is true, +// _fetchUserStreams retrieves all streams for the given user. If makeIfNotExists is true, // a stream will be made for this user if one doesn't exist and it will be returned. This // function does not wait for data to be available on the stream. -// NB: Callers should have locked the mutex before calling this function. -func (n *Notifier) fetchUserStreams(userID string) []*UserDeviceStream { +func (n *Notifier) _fetchUserStreams(userID string) []*UserDeviceStream { user, ok := n.userDeviceStreams[userID] if !ok { return []*UserDeviceStream{} } - streams := []*UserDeviceStream{} + streams := make([]*UserDeviceStream, 0, len(user)) for _, stream := range user { streams = append(streams, stream) } return streams } -// Not thread-safe: must be called on the OnNewEvent goroutine only -func (n *Notifier) addJoinedUser(roomID, userID string) { +func (n *Notifier) _addJoinedUser(roomID, userID string) { if _, ok := n.roomIDToJoinedUsers[roomID]; !ok { n.roomIDToJoinedUsers[roomID] = make(userIDSet) } n.roomIDToJoinedUsers[roomID].add(userID) } -// Not thread-safe: must be called on the OnNewEvent goroutine only -func (n *Notifier) removeJoinedUser(roomID, userID string) { +func (n *Notifier) _removeJoinedUser(roomID, userID string) { if _, ok := n.roomIDToJoinedUsers[roomID]; !ok { n.roomIDToJoinedUsers[roomID] = make(userIDSet) } n.roomIDToJoinedUsers[roomID].remove(userID) } -// Not thread-safe: must be called on the OnNewEvent goroutine only -func (n *Notifier) joinedUsers(roomID string) (userIDs []string) { +func (n *Notifier) JoinedUsers(roomID string) (userIDs []string) { + n.lock.RLock() + defer n.lock.RUnlock() + return n._joinedUsers(roomID) +} + +func (n *Notifier) _joinedUsers(roomID string) (userIDs []string) { if _, ok := n.roomIDToJoinedUsers[roomID]; !ok { return } return n.roomIDToJoinedUsers[roomID].values() } -// Not thread-safe: must be called on the OnNewEvent goroutine only -func (n *Notifier) addPeekingDevice(roomID, userID, deviceID string) { +func (n *Notifier) _addPeekingDevice(roomID, userID, deviceID string) { if _, ok := n.roomIDToPeekingDevices[roomID]; !ok { n.roomIDToPeekingDevices[roomID] = make(peekingDeviceSet) } n.roomIDToPeekingDevices[roomID].add(types.PeekingDevice{UserID: userID, DeviceID: deviceID}) } -// Not thread-safe: must be called on the OnNewEvent goroutine only -// nolint:unused -func (n *Notifier) removePeekingDevice(roomID, userID, deviceID string) { +func (n *Notifier) _removePeekingDevice(roomID, userID, deviceID string) { if _, ok := n.roomIDToPeekingDevices[roomID]; !ok { n.roomIDToPeekingDevices[roomID] = make(peekingDeviceSet) } @@ -417,22 +480,26 @@ func (n *Notifier) removePeekingDevice(roomID, userID, deviceID string) { n.roomIDToPeekingDevices[roomID].remove(types.PeekingDevice{UserID: userID, DeviceID: deviceID}) } -// Not thread-safe: must be called on the OnNewEvent goroutine only func (n *Notifier) PeekingDevices(roomID string) (peekingDevices []types.PeekingDevice) { + n.lock.RLock() + defer n.lock.RUnlock() + return n._peekingDevices(roomID) +} + +func (n *Notifier) _peekingDevices(roomID string) (peekingDevices []types.PeekingDevice) { if _, ok := n.roomIDToPeekingDevices[roomID]; !ok { return } return n.roomIDToPeekingDevices[roomID].values() } -// removeEmptyUserStreams iterates through the user stream map and removes any +// _removeEmptyUserStreams iterates through the user stream map and removes any // that have been empty for a certain amount of time. This is a crude way of // ensuring that the userStreams map doesn't grow forver. // This should be called when the notifier gets called for whatever reason, // the function itself is responsible for ensuring it doesn't iterate too // often. -// NB: Callers should have locked the mutex before calling this function. -func (n *Notifier) removeEmptyUserStreams() { +func (n *Notifier) _removeEmptyUserStreams() { // Only clean up now and again now := time.Now() if n.lastCleanUpTime.Add(time.Minute).After(now) { @@ -454,10 +521,10 @@ func (n *Notifier) removeEmptyUserStreams() { } // A string set, mainly existing for improving clarity of structs in this file. -type userIDSet map[string]bool +type userIDSet map[string]struct{} func (s userIDSet) add(str string) { - s[str] = true + s[str] = struct{}{} } func (s userIDSet) remove(str string) { @@ -465,6 +532,7 @@ func (s userIDSet) remove(str string) { } func (s userIDSet) values() (vals []string) { + vals = make([]string, 0, len(s)) for str := range s { vals = append(vals, str) } @@ -473,10 +541,10 @@ func (s userIDSet) values() (vals []string) { // A set of PeekingDevices, similar to userIDSet -type peekingDeviceSet map[types.PeekingDevice]bool +type peekingDeviceSet map[types.PeekingDevice]struct{} func (s peekingDeviceSet) add(d types.PeekingDevice) { - s[d] = true + s[d] = struct{}{} } // nolint:unused @@ -485,6 +553,7 @@ func (s peekingDeviceSet) remove(d types.PeekingDevice) { } func (s peekingDeviceSet) values() (vals []types.PeekingDevice) { + vals = make([]types.PeekingDevice, 0, len(s)) for d := range s { vals = append(vals, d) } diff --git a/syncapi/notifier/notifier_test.go b/syncapi/notifier/notifier_test.go index 60403d5d5..b06313712 100644 --- a/syncapi/notifier/notifier_test.go +++ b/syncapi/notifier/notifier_test.go @@ -107,7 +107,8 @@ func mustEqualPositions(t *testing.T, got, want types.StreamingToken) { // Test that the current position is returned if a request is already behind. func TestImmediateNotification(t *testing.T) { - n := NewNotifier(syncPositionBefore) + n := NewNotifier() + n.SetCurrentPosition(syncPositionBefore) pos, err := waitForEvents(n, newTestSyncRequest(alice, aliceDev, syncPositionVeryOld)) if err != nil { t.Fatalf("TestImmediateNotification error: %s", err) @@ -117,7 +118,8 @@ func TestImmediateNotification(t *testing.T) { // Test that new events to a joined room unblocks the request. func TestNewEventAndJoinedToRoom(t *testing.T) { - n := NewNotifier(syncPositionBefore) + n := NewNotifier() + n.SetCurrentPosition(syncPositionBefore) n.setUsersJoinedToRooms(map[string][]string{ roomID: {alice, bob}, }) @@ -142,7 +144,8 @@ func TestNewEventAndJoinedToRoom(t *testing.T) { } func TestCorrectStream(t *testing.T) { - n := NewNotifier(syncPositionBefore) + n := NewNotifier() + n.SetCurrentPosition(syncPositionBefore) stream := lockedFetchUserStream(n, bob, bobDev) if stream.UserID != bob { t.Fatalf("expected user %q, got %q", bob, stream.UserID) @@ -153,7 +156,8 @@ func TestCorrectStream(t *testing.T) { } func TestCorrectStreamWakeup(t *testing.T) { - n := NewNotifier(syncPositionBefore) + n := NewNotifier() + n.SetCurrentPosition(syncPositionBefore) awoken := make(chan string) streamone := lockedFetchUserStream(n, alice, "one") @@ -161,9 +165,9 @@ func TestCorrectStreamWakeup(t *testing.T) { go func() { select { - case <-streamone.signalChannel: + case <-streamone.ch(): awoken <- "one" - case <-streamtwo.signalChannel: + case <-streamtwo.ch(): awoken <- "two" } }() @@ -171,7 +175,7 @@ func TestCorrectStreamWakeup(t *testing.T) { time.Sleep(1 * time.Second) wake := "two" - n.wakeupUserDevice(alice, []string{wake}, syncPositionAfter) + n._wakeupUserDevice(alice, []string{wake}, syncPositionAfter) if result := <-awoken; result != wake { t.Fatalf("expected to wake %q, got %q", wake, result) @@ -180,7 +184,8 @@ func TestCorrectStreamWakeup(t *testing.T) { // Test that an invite unblocks the request func TestNewInviteEventForUser(t *testing.T) { - n := NewNotifier(syncPositionBefore) + n := NewNotifier() + n.SetCurrentPosition(syncPositionBefore) n.setUsersJoinedToRooms(map[string][]string{ roomID: {alice, bob}, }) @@ -236,7 +241,8 @@ func TestEDUWakeup(t *testing.T) { // Test that all blocked requests get woken up on a new event. func TestMultipleRequestWakeup(t *testing.T) { - n := NewNotifier(syncPositionBefore) + n := NewNotifier() + n.SetCurrentPosition(syncPositionBefore) n.setUsersJoinedToRooms(map[string][]string{ roomID: {alice, bob}, }) @@ -272,7 +278,8 @@ func TestMultipleRequestWakeup(t *testing.T) { func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { // listen as bob. Make bob leave room. Make alice send event to room. // Make sure alice gets woken up only and not bob as well. - n := NewNotifier(syncPositionBefore) + n := NewNotifier() + n.SetCurrentPosition(syncPositionBefore) n.setUsersJoinedToRooms(map[string][]string{ roomID: {alice, bob}, }) @@ -352,10 +359,10 @@ func waitForBlocking(s *UserDeviceStream, numBlocking uint) { // lockedFetchUserStream invokes Notifier.fetchUserStream, respecting Notifier.streamLock. // A new stream is made if it doesn't exist already. func lockedFetchUserStream(n *Notifier, userID, deviceID string) *UserDeviceStream { - n.streamLock.Lock() - defer n.streamLock.Unlock() + n.lock.Lock() + defer n.lock.Unlock() - return n.fetchUserDeviceStream(userID, deviceID, true) + return n._fetchUserDeviceStream(userID, deviceID, true) } func newTestSyncRequest(userID, deviceID string, since types.StreamingToken) types.SyncRequest { diff --git a/syncapi/notifier/userstream.go b/syncapi/notifier/userstream.go index 720185d52..bcd69fad5 100644 --- a/syncapi/notifier/userstream.go +++ b/syncapi/notifier/userstream.go @@ -118,6 +118,12 @@ func (s *UserDeviceStream) TimeOfLastNonEmpty() time.Time { return s.timeOfLastChannel } +func (s *UserDeviceStream) ch() <-chan struct{} { + s.lock.Lock() + defer s.lock.Unlock() + return s.signalChannel +} + // GetSyncPosition returns last sync position which the UserStream was // notified about func (s *UserDeviceStreamListener) GetSyncPosition() types.StreamingToken { diff --git a/syncapi/producers/federationapi_presence.go b/syncapi/producers/federationapi_presence.go new file mode 100644 index 000000000..dc03457e3 --- /dev/null +++ b/syncapi/producers/federationapi_presence.go @@ -0,0 +1,48 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package producers + +import ( + "strconv" + "time" + + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/gomatrixserverlib" + "github.com/nats-io/nats.go" +) + +// FederationAPIPresenceProducer produces events for the federation API server to consume +type FederationAPIPresenceProducer struct { + Topic string + JetStream nats.JetStreamContext +} + +func (f *FederationAPIPresenceProducer) SendPresence( + userID string, presence types.Presence, statusMsg *string, +) error { + msg := nats.NewMsg(f.Topic) + msg.Header.Set(jetstream.UserID, userID) + msg.Header.Set("presence", presence.String()) + msg.Header.Set("from_sync", "true") // only update last_active_ts and presence + msg.Header.Set("last_active_ts", strconv.Itoa(int(gomatrixserverlib.AsTimestamp(time.Now())))) + + if statusMsg != nil { + msg.Header.Set("status_msg", *statusMsg) + } + + _, err := f.JetStream.PublishMsg(msg) + return err +} diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index b6ac5be19..cf3fd5532 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -17,7 +17,6 @@ package storage import ( "context" - eduAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" @@ -27,6 +26,7 @@ import ( ) type Database interface { + Presence MaxStreamPositionForPDUs(ctx context.Context) (types.StreamPosition, error) MaxStreamPositionForReceipts(ctx context.Context) (types.StreamPosition, error) MaxStreamPositionForInvites(ctx context.Context) (types.StreamPosition, error) @@ -38,6 +38,7 @@ type Database interface { GetStateDeltasForFullStateSync(ctx context.Context, device *userapi.Device, r types.Range, userID string, stateFilter *gomatrixserverlib.StateFilter) ([]types.StateDelta, []string, error) GetStateDeltas(ctx context.Context, device *userapi.Device, r types.Range, userID string, stateFilter *gomatrixserverlib.StateFilter) ([]types.StateDelta, []string, error) RoomIDsWithMembership(ctx context.Context, userID string, membership string) ([]string, error) + MembershipCount(ctx context.Context, roomID, membership string, pos types.StreamPosition) (int, error) RecentEvents(ctx context.Context, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error) @@ -46,7 +47,7 @@ type Database interface { InviteEventsInRange(ctx context.Context, targetUserID string, r types.Range) (map[string]*gomatrixserverlib.HeaderedEvent, map[string]*gomatrixserverlib.HeaderedEvent, error) PeeksInRange(ctx context.Context, userID, deviceID string, r types.Range) (peeks []types.Peek, err error) - RoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []eduAPI.OutputReceiptEvent, error) + RoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []types.OutputReceiptEvent, error) // AllJoinedUsersInRooms returns a map of room ID to a list of all joined user IDs. AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) @@ -103,7 +104,7 @@ type Database interface { // DeletePeek deletes all peeks for a given room by a given user // Returns an error if there was a problem communicating with the database. DeletePeeks(ctx context.Context, RoomID, UserID string) (types.StreamPosition, error) - // GetEventsInTopologicalRange retrieves all of the events on a given ordering using the given extremities and limit. + // GetEventsInTopologicalRange retrieves all of the events on a given ordering using the given extremities and limit. If backwardsOrdering is true, the most recent event must be first, else last. GetEventsInTopologicalRange(ctx context.Context, from, to *types.TopologyToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error) // EventPositionInTopology returns the depth and stream position of the given event. EventPositionInTopology(ctx context.Context, eventID string) (types.TopologyToken, error) @@ -136,7 +137,7 @@ type Database interface { // StoreReceipt stores new receipt events StoreReceipt(ctx context.Context, roomId, receiptType, userId, eventId string, timestamp gomatrixserverlib.Timestamp) (pos types.StreamPosition, err error) // GetRoomReceipts gets all receipts for a given roomID - GetRoomReceipts(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) ([]eduAPI.OutputReceiptEvent, error) + GetRoomReceipts(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) ([]types.OutputReceiptEvent, error) // UpsertRoomUnreadNotificationCounts updates the notification statistics about a (user, room) key. UpsertRoomUnreadNotificationCounts(ctx context.Context, userID, roomID string, notificationCount, highlightCount int) (types.StreamPosition, error) @@ -149,4 +150,14 @@ type Database interface { SelectContextAfterEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error) StreamToTopologicalPosition(ctx context.Context, roomID string, streamPos types.StreamPosition, backwardOrdering bool) (types.TopologyToken, error) + + IgnoresForUser(ctx context.Context, userID string) (*types.IgnoredUsers, error) + UpdateIgnoresForUser(ctx context.Context, userID string, ignores *types.IgnoredUsers) error +} + +type Presence interface { + UpdatePresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (types.StreamPosition, error) + GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error) + PresenceAfter(ctx context.Context, after types.StreamPosition) (map[string]*types.PresenceInternal, error) + MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error) } diff --git a/syncapi/storage/postgres/backwards_extremities_table.go b/syncapi/storage/postgres/backwards_extremities_table.go index d5cf563a6..d4515735c 100644 --- a/syncapi/storage/postgres/backwards_extremities_table.go +++ b/syncapi/storage/postgres/backwards_extremities_table.go @@ -47,14 +47,10 @@ const selectBackwardExtremitiesForRoomSQL = "" + const deleteBackwardExtremitySQL = "" + "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" -const deleteBackwardExtremitiesForRoomSQL = "" + - "DELETE FROM syncapi_backward_extremities WHERE room_id = $1" - type backwardExtremitiesStatements struct { insertBackwardExtremityStmt *sql.Stmt selectBackwardExtremitiesForRoomStmt *sql.Stmt deleteBackwardExtremityStmt *sql.Stmt - deleteBackwardExtremitiesForRoomStmt *sql.Stmt } func NewPostgresBackwardsExtremitiesTable(db *sql.DB) (tables.BackwardsExtremities, error) { @@ -72,9 +68,6 @@ func NewPostgresBackwardsExtremitiesTable(db *sql.DB) (tables.BackwardsExtremiti if s.deleteBackwardExtremityStmt, err = db.Prepare(deleteBackwardExtremitySQL); err != nil { return nil, err } - if s.deleteBackwardExtremitiesForRoomStmt, err = db.Prepare(deleteBackwardExtremitiesForRoomSQL); err != nil { - return nil, err - } return s, nil } @@ -113,10 +106,3 @@ func (s *backwardExtremitiesStatements) DeleteBackwardExtremity( _, err = sqlutil.TxStmt(txn, s.deleteBackwardExtremityStmt).ExecContext(ctx, roomID, knownEventID) return } - -func (s *backwardExtremitiesStatements) DeleteBackwardExtremitiesForRoom( - ctx context.Context, txn *sql.Tx, roomID string, -) (err error) { - _, err = sqlutil.TxStmt(txn, s.deleteBackwardExtremitiesForRoomStmt).ExecContext(ctx, roomID) - return err -} diff --git a/syncapi/storage/postgres/ignores_table.go b/syncapi/storage/postgres/ignores_table.go new file mode 100644 index 000000000..055a1a237 --- /dev/null +++ b/syncapi/storage/postgres/ignores_table.go @@ -0,0 +1,87 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgres + +import ( + "context" + "database/sql" + "encoding/json" + + "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/types" +) + +const ignoresSchema = ` +-- Stores data about ignoress +CREATE TABLE IF NOT EXISTS syncapi_ignores ( + -- The user ID whose ignore list this belongs to. + user_id TEXT NOT NULL, + ignores_json TEXT NOT NULL, + PRIMARY KEY(user_id) +); +` + +const selectIgnoresSQL = "" + + "SELECT ignores_json FROM syncapi_ignores WHERE user_id = $1" + +const upsertIgnoresSQL = "" + + "INSERT INTO syncapi_ignores (user_id, ignores_json) VALUES ($1, $2)" + + " ON CONFLICT (user_id) DO UPDATE set ignores_json = $2" + +type ignoresStatements struct { + selectIgnoresStmt *sql.Stmt + upsertIgnoresStmt *sql.Stmt +} + +func NewPostgresIgnoresTable(db *sql.DB) (tables.Ignores, error) { + _, err := db.Exec(ignoresSchema) + if err != nil { + return nil, err + } + s := &ignoresStatements{} + if s.selectIgnoresStmt, err = db.Prepare(selectIgnoresSQL); err != nil { + return nil, err + } + if s.upsertIgnoresStmt, err = db.Prepare(upsertIgnoresSQL); err != nil { + return nil, err + } + return s, nil +} + +func (s *ignoresStatements) SelectIgnores( + ctx context.Context, userID string, +) (*types.IgnoredUsers, error) { + var ignoresData []byte + err := s.selectIgnoresStmt.QueryRowContext(ctx, userID).Scan(&ignoresData) + if err != nil { + return nil, err + } + var ignores types.IgnoredUsers + if err = json.Unmarshal(ignoresData, &ignores); err != nil { + return nil, err + } + return &ignores, nil +} + +func (s *ignoresStatements) UpsertIgnores( + ctx context.Context, userID string, ignores *types.IgnoredUsers, +) error { + ignoresJSON, err := json.Marshal(ignores) + if err != nil { + return err + } + _, err = s.upsertIgnoresStmt.ExecContext(ctx, userID, ignoresJSON) + return err +} diff --git a/syncapi/storage/postgres/memberships_table.go b/syncapi/storage/postgres/memberships_table.go index 6566544d6..39fa656cb 100644 --- a/syncapi/storage/postgres/memberships_table.go +++ b/syncapi/storage/postgres/memberships_table.go @@ -56,15 +56,14 @@ const upsertMembershipSQL = "" + " ON CONFLICT ON CONSTRAINT syncapi_memberships_unique" + " DO UPDATE SET event_id = $4, stream_pos = $5, topological_pos = $6" -const selectMembershipSQL = "" + - "SELECT event_id, stream_pos, topological_pos FROM syncapi_memberships" + - " WHERE room_id = $1 AND user_id = $2 AND membership = ANY($3)" + - " ORDER BY stream_pos DESC" + - " LIMIT 1" +const selectMembershipCountSQL = "" + + "SELECT COUNT(*) FROM (" + + " SELECT DISTINCT ON (room_id, user_id) room_id, user_id, membership FROM syncapi_memberships WHERE room_id = $1 AND stream_pos <= $2 ORDER BY room_id, user_id, stream_pos DESC" + + ") t WHERE t.membership = $3" type membershipsStatements struct { - upsertMembershipStmt *sql.Stmt - selectMembershipStmt *sql.Stmt + upsertMembershipStmt *sql.Stmt + selectMembershipCountStmt *sql.Stmt } func NewPostgresMembershipsTable(db *sql.DB) (tables.Memberships, error) { @@ -76,7 +75,7 @@ func NewPostgresMembershipsTable(db *sql.DB) (tables.Memberships, error) { if s.upsertMembershipStmt, err = db.Prepare(upsertMembershipSQL); err != nil { return nil, err } - if s.selectMembershipStmt, err = db.Prepare(selectMembershipSQL); err != nil { + if s.selectMembershipCountStmt, err = db.Prepare(selectMembershipCountSQL); err != nil { return nil, err } return s, nil @@ -102,10 +101,10 @@ func (s *membershipsStatements) UpsertMembership( return err } -func (s *membershipsStatements) SelectMembership( - ctx context.Context, txn *sql.Tx, roomID, userID, memberships []string, -) (eventID string, streamPos, topologyPos types.StreamPosition, err error) { - stmt := sqlutil.TxStmt(txn, s.selectMembershipStmt) - err = stmt.QueryRowContext(ctx, roomID, userID, memberships).Scan(&eventID, &streamPos, &topologyPos) +func (s *membershipsStatements) SelectMembershipCount( + ctx context.Context, txn *sql.Tx, roomID, membership string, pos types.StreamPosition, +) (count int, err error) { + stmt := sqlutil.TxStmt(txn, s.selectMembershipCountStmt) + err = stmt.QueryRowContext(ctx, roomID, pos, membership).Scan(&count) return } diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index 14af6a949..a30e220ba 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -427,7 +427,7 @@ func (s *outputRoomEventsStatements) SelectEarlyEvents( // selectEvents returns the events for the given event IDs. If an event is // missing from the database, it will be omitted. func (s *outputRoomEventsStatements) SelectEvents( - ctx context.Context, txn *sql.Tx, eventIDs []string, + ctx context.Context, txn *sql.Tx, eventIDs []string, preserveOrder bool, ) ([]types.StreamEvent, error) { stmt := sqlutil.TxStmt(txn, s.selectEventsStmt) rows, err := stmt.QueryContext(ctx, pq.StringArray(eventIDs)) @@ -435,7 +435,25 @@ func (s *outputRoomEventsStatements) SelectEvents( return nil, err } defer internal.CloseAndLogIfError(ctx, rows, "selectEvents: rows.close() failed") - return rowsToStreamEvents(rows) + streamEvents, err := rowsToStreamEvents(rows) + if err != nil { + return nil, err + } + if preserveOrder { + eventMap := make(map[string]types.StreamEvent) + for _, ev := range streamEvents { + eventMap[ev.EventID()] = ev + } + var returnEvents []types.StreamEvent + for _, eventID := range eventIDs { + ev, ok := eventMap[eventID] + if ok { + returnEvents = append(returnEvents, ev) + } + } + return returnEvents, nil + } + return streamEvents, nil } func (s *outputRoomEventsStatements) DeleteEventsForRoom( diff --git a/syncapi/storage/postgres/output_room_events_topology_table.go b/syncapi/storage/postgres/output_room_events_topology_table.go index 626386ba0..a1fc9b2a3 100644 --- a/syncapi/storage/postgres/output_room_events_topology_table.go +++ b/syncapi/storage/postgres/output_room_events_topology_table.go @@ -73,9 +73,6 @@ const selectMaxPositionInTopologySQL = "" + "SELECT MAX(topological_position) FROM syncapi_output_room_events_topology WHERE room_id=$1" + ") ORDER BY stream_position DESC LIMIT 1" -const deleteTopologyForRoomSQL = "" + - "DELETE FROM syncapi_output_room_events_topology WHERE room_id = $1" - const selectStreamToTopologicalPositionAscSQL = "" + "SELECT topological_position FROM syncapi_output_room_events_topology WHERE room_id = $1 AND stream_position >= $2 ORDER BY topological_position ASC LIMIT 1;" @@ -88,7 +85,6 @@ type outputRoomEventsTopologyStatements struct { selectEventIDsInRangeDESCStmt *sql.Stmt selectPositionInTopologyStmt *sql.Stmt selectMaxPositionInTopologyStmt *sql.Stmt - deleteTopologyForRoomStmt *sql.Stmt selectStreamToTopologicalPositionAscStmt *sql.Stmt selectStreamToTopologicalPositionDescStmt *sql.Stmt } @@ -114,9 +110,6 @@ func NewPostgresTopologyTable(db *sql.DB) (tables.Topology, error) { if s.selectMaxPositionInTopologyStmt, err = db.Prepare(selectMaxPositionInTopologySQL); err != nil { return nil, err } - if s.deleteTopologyForRoomStmt, err = db.Prepare(deleteTopologyForRoomSQL); err != nil { - return nil, err - } if s.selectStreamToTopologicalPositionAscStmt, err = db.Prepare(selectStreamToTopologicalPositionAscSQL); err != nil { return nil, err } @@ -148,9 +141,9 @@ func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( // is requested or not. var stmt *sql.Stmt if chronologicalOrder { - stmt = s.selectEventIDsInRangeASCStmt + stmt = sqlutil.TxStmt(txn, s.selectEventIDsInRangeASCStmt) } else { - stmt = s.selectEventIDsInRangeDESCStmt + stmt = sqlutil.TxStmt(txn, s.selectEventIDsInRangeDESCStmt) } // Query the event IDs. @@ -203,10 +196,3 @@ func (s *outputRoomEventsTopologyStatements) SelectMaxPositionInTopology( err = s.selectMaxPositionInTopologyStmt.QueryRowContext(ctx, roomID).Scan(&pos, &spos) return } - -func (s *outputRoomEventsTopologyStatements) DeleteTopologyForRoom( - ctx context.Context, txn *sql.Tx, roomID string, -) (err error) { - _, err = sqlutil.TxStmt(txn, s.deleteTopologyForRoomStmt).ExecContext(ctx, roomID) - return err -} diff --git a/syncapi/storage/postgres/presence_table.go b/syncapi/storage/postgres/presence_table.go new file mode 100644 index 000000000..49336c4eb --- /dev/null +++ b/syncapi/storage/postgres/presence_table.go @@ -0,0 +1,162 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgres + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/gomatrixserverlib" +) + +const presenceSchema = ` +CREATE SEQUENCE IF NOT EXISTS syncapi_presence_id; +-- Stores data about presence +CREATE TABLE IF NOT EXISTS syncapi_presence ( + -- The ID + id BIGINT PRIMARY KEY DEFAULT nextval('syncapi_presence_id'), + -- The Matrix user ID + user_id TEXT NOT NULL, + -- The actual presence + presence INT NOT NULL, + -- The status message + status_msg TEXT, + -- The last time an action was received by this user + last_active_ts BIGINT NOT NULL, + CONSTRAINT presence_presences_unique UNIQUE (user_id) +); +CREATE INDEX IF NOT EXISTS syncapi_presence_user_id ON syncapi_presence(user_id); +` + +const upsertPresenceSQL = "" + + "INSERT INTO syncapi_presence AS p" + + " (user_id, presence, status_msg, last_active_ts)" + + " VALUES ($1, $2, $3, $4)" + + " ON CONFLICT (user_id)" + + " DO UPDATE SET id = nextval('syncapi_presence_id')," + + " presence = $2, status_msg = COALESCE($3, p.status_msg), last_active_ts = $4" + + " RETURNING id" + +const upsertPresenceFromSyncSQL = "" + + "INSERT INTO syncapi_presence AS p" + + " (user_id, presence, last_active_ts)" + + " VALUES ($1, $2, $3)" + + " ON CONFLICT (user_id)" + + " DO UPDATE SET id = nextval('syncapi_presence_id')," + + " presence = $2, last_active_ts = $3" + + " RETURNING id" + +const selectPresenceForUserSQL = "" + + "SELECT presence, status_msg, last_active_ts" + + " FROM syncapi_presence" + + " WHERE user_id = $1 LIMIT 1" + +const selectMaxPresenceSQL = "" + + "SELECT COALESCE(MAX(id), 0) FROM syncapi_presence" + +const selectPresenceAfter = "" + + " SELECT id, user_id, presence, status_msg, last_active_ts" + + " FROM syncapi_presence" + + " WHERE id > $1" + +type presenceStatements struct { + upsertPresenceStmt *sql.Stmt + upsertPresenceFromSyncStmt *sql.Stmt + selectPresenceForUsersStmt *sql.Stmt + selectMaxPresenceStmt *sql.Stmt + selectPresenceAfterStmt *sql.Stmt +} + +func NewPostgresPresenceTable(db *sql.DB) (*presenceStatements, error) { + _, err := db.Exec(presenceSchema) + if err != nil { + return nil, err + } + s := &presenceStatements{} + return s, sqlutil.StatementList{ + {&s.upsertPresenceStmt, upsertPresenceSQL}, + {&s.upsertPresenceFromSyncStmt, upsertPresenceFromSyncSQL}, + {&s.selectPresenceForUsersStmt, selectPresenceForUserSQL}, + {&s.selectMaxPresenceStmt, selectMaxPresenceSQL}, + {&s.selectPresenceAfterStmt, selectPresenceAfter}, + }.Prepare(db) +} + +// UpsertPresence creates/updates a presence status. +func (p *presenceStatements) UpsertPresence( + ctx context.Context, + txn *sql.Tx, + userID string, + statusMsg *string, + presence types.Presence, + lastActiveTS gomatrixserverlib.Timestamp, + fromSync bool, +) (pos types.StreamPosition, err error) { + if fromSync { + stmt := sqlutil.TxStmt(txn, p.upsertPresenceFromSyncStmt) + err = stmt.QueryRowContext(ctx, userID, presence, lastActiveTS).Scan(&pos) + } else { + stmt := sqlutil.TxStmt(txn, p.upsertPresenceStmt) + err = stmt.QueryRowContext(ctx, userID, presence, statusMsg, lastActiveTS).Scan(&pos) + } + return +} + +// GetPresenceForUser returns the current presence of a user. +func (p *presenceStatements) GetPresenceForUser( + ctx context.Context, txn *sql.Tx, + userID string, +) (*types.PresenceInternal, error) { + result := &types.PresenceInternal{ + UserID: userID, + } + stmt := sqlutil.TxStmt(txn, p.selectPresenceForUsersStmt) + err := stmt.QueryRowContext(ctx, userID).Scan(&result.Presence, &result.ClientFields.StatusMsg, &result.LastActiveTS) + result.ClientFields.Presence = result.Presence.String() + return result, err +} + +func (p *presenceStatements) GetMaxPresenceID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { + stmt := sqlutil.TxStmt(txn, p.selectMaxPresenceStmt) + err = stmt.QueryRowContext(ctx).Scan(&pos) + return +} + +// GetPresenceAfter returns the changes presences after a given stream id +func (p *presenceStatements) GetPresenceAfter( + ctx context.Context, txn *sql.Tx, + after types.StreamPosition, +) (presences map[string]*types.PresenceInternal, err error) { + presences = make(map[string]*types.PresenceInternal) + stmt := sqlutil.TxStmt(txn, p.selectPresenceAfterStmt) + + rows, err := stmt.QueryContext(ctx, after) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "GetPresenceAfter: failed to close rows") + for rows.Next() { + qryRes := &types.PresenceInternal{} + if err := rows.Scan(&qryRes.StreamPos, &qryRes.UserID, &qryRes.Presence, &qryRes.ClientFields.StatusMsg, &qryRes.LastActiveTS); err != nil { + return nil, err + } + qryRes.ClientFields.Presence = qryRes.Presence.String() + presences[qryRes.UserID] = qryRes + } + return presences, rows.Err() +} diff --git a/syncapi/storage/postgres/receipt_table.go b/syncapi/storage/postgres/receipt_table.go index 474d0c020..2a42ffd74 100644 --- a/syncapi/storage/postgres/receipt_table.go +++ b/syncapi/storage/postgres/receipt_table.go @@ -21,7 +21,6 @@ import ( "github.com/lib/pq" - "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" @@ -95,16 +94,16 @@ func (r *receiptStatements) UpsertReceipt(ctx context.Context, txn *sql.Tx, room return } -func (r *receiptStatements) SelectRoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []api.OutputReceiptEvent, error) { +func (r *receiptStatements) SelectRoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []types.OutputReceiptEvent, error) { var lastPos types.StreamPosition rows, err := r.selectRoomReceipts.QueryContext(ctx, pq.Array(roomIDs), streamPos) if err != nil { return 0, nil, fmt.Errorf("unable to query room receipts: %w", err) } defer internal.CloseAndLogIfError(ctx, rows, "SelectRoomReceiptsAfter: rows.close() failed") - var res []api.OutputReceiptEvent + var res []types.OutputReceiptEvent for rows.Next() { - r := api.OutputReceiptEvent{} + r := types.OutputReceiptEvent{} var id types.StreamPosition err = rows.Scan(&id, &r.RoomID, &r.Type, &r.UserID, &r.EventID, &r.Timestamp) if err != nil { diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index 4e4b5c0bb..b0382512a 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -90,6 +90,14 @@ func NewDatabase(dbProperties *config.DatabaseOptions) (*SyncServerDatasource, e if err != nil { return nil, err } + ignores, err := NewPostgresIgnoresTable(d.db) + if err != nil { + return nil, err + } + presence, err := NewPostgresPresenceTable(d.db) + if err != nil { + return nil, err + } m := sqlutil.NewMigrations() deltas.LoadFixSequences(m) deltas.LoadRemoveSendToDeviceSentColumn(m) @@ -111,6 +119,8 @@ func NewDatabase(dbProperties *config.DatabaseOptions) (*SyncServerDatasource, e Receipts: receipts, Memberships: memberships, NotificationData: notificationData, + Ignores: ignores, + Presence: presence, } return &d, nil } diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index 9a2dc0d44..14db5795c 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -20,7 +20,6 @@ import ( "encoding/json" "fmt" - eduAPI "github.com/matrix-org/dendrite/eduserver/api" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/internal/eventutil" @@ -49,6 +48,8 @@ type Database struct { Receipts tables.Receipts Memberships tables.Memberships NotificationData tables.NotificationData + Ignores tables.Ignores + Presence tables.Presence } func (d *Database) readOnlySnapshot(ctx context.Context) (*sql.Tx, error) { @@ -119,6 +120,10 @@ func (d *Database) RoomIDsWithMembership(ctx context.Context, userID string, mem return d.CurrentRoomState.SelectRoomIDsWithMembership(ctx, nil, userID, membership) } +func (d *Database) MembershipCount(ctx context.Context, roomID, membership string, pos types.StreamPosition) (int, error) { + return d.Memberships.SelectMembershipCount(ctx, nil, roomID, membership, pos) +} + func (d *Database) RecentEvents(ctx context.Context, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error) { return d.OutputEvents.SelectRecentEvents(ctx, nil, roomID, r, eventFilter, chronologicalOrder, onlySyncEvents) } @@ -135,7 +140,7 @@ func (d *Database) PeeksInRange(ctx context.Context, userID, deviceID string, r return d.Peeks.SelectPeeksInRange(ctx, nil, userID, deviceID, r) } -func (d *Database) RoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []eduAPI.OutputReceiptEvent, error) { +func (d *Database) RoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []types.OutputReceiptEvent, error) { return d.Receipts.SelectRoomReceiptsAfter(ctx, roomIDs, streamPos) } @@ -145,7 +150,7 @@ func (d *Database) RoomReceiptsAfter(ctx context.Context, roomIDs []string, stre // Returns an error if there was a problem talking with the database. // Does not include any transaction IDs in the returned events. func (d *Database) Events(ctx context.Context, eventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error) { - streamEvents, err := d.OutputEvents.SelectEvents(ctx, nil, eventIDs) + streamEvents, err := d.OutputEvents.SelectEvents(ctx, nil, eventIDs, false) if err != nil { return nil, err } @@ -307,7 +312,7 @@ func (d *Database) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, e // Check if we have all of the event's previous events. If an event is // missing, add it to the room's backward extremities. - prevEvents, err := d.OutputEvents.SelectEvents(ctx, txn, ev.PrevEventIDs()) + prevEvents, err := d.OutputEvents.SelectEvents(ctx, txn, ev.PrevEventIDs(), false) if err != nil { return err } @@ -452,7 +457,7 @@ func (d *Database) GetEventsInTopologicalRange( } // Retrieve the events' contents using their IDs. - events, err = d.OutputEvents.SelectEvents(ctx, nil, eIDs) + events, err = d.OutputEvents.SelectEvents(ctx, nil, eIDs, true) return } @@ -614,7 +619,7 @@ func (d *Database) fetchMissingStateEvents( ) ([]types.StreamEvent, error) { // Fetch from the events table first so we pick up the stream ID for the // event. - events, err := d.OutputEvents.SelectEvents(ctx, txn, eventIDs) + events, err := d.OutputEvents.SelectEvents(ctx, txn, eventIDs, false) if err != nil { return nil, err } @@ -972,7 +977,7 @@ func (d *Database) StoreReceipt(ctx context.Context, roomId, receiptType, userId return } -func (d *Database) GetRoomReceipts(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) ([]eduAPI.OutputReceiptEvent, error) { +func (d *Database) GetRoomReceipts(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) ([]types.OutputReceiptEvent, error) { _, receipts, err := d.Receipts.SelectRoomReceiptsAfter(ctx, roomIDs, streamPos) return receipts, err } @@ -999,3 +1004,27 @@ func (s *Database) SelectContextBeforeEvent(ctx context.Context, id int, roomID func (s *Database) SelectContextAfterEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error) { return s.OutputEvents.SelectContextAfterEvent(ctx, nil, id, roomID, filter) } + +func (s *Database) IgnoresForUser(ctx context.Context, userID string) (*types.IgnoredUsers, error) { + return s.Ignores.SelectIgnores(ctx, userID) +} + +func (s *Database) UpdateIgnoresForUser(ctx context.Context, userID string, ignores *types.IgnoredUsers) error { + return s.Ignores.UpsertIgnores(ctx, userID, ignores) +} + +func (s *Database) UpdatePresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (types.StreamPosition, error) { + return s.Presence.UpsertPresence(ctx, nil, userID, statusMsg, presence, lastActiveTS, fromSync) +} + +func (s *Database) GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error) { + return s.Presence.GetPresenceForUser(ctx, nil, userID) +} + +func (s *Database) PresenceAfter(ctx context.Context, after types.StreamPosition) (map[string]*types.PresenceInternal, error) { + return s.Presence.GetPresenceAfter(ctx, nil, after) +} + +func (s *Database) MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error) { + return s.Presence.GetMaxPresenceID(ctx, nil) +} diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go index 24c442240..5b2287e6d 100644 --- a/syncapi/storage/sqlite3/account_data_table.go +++ b/syncapi/storage/sqlite3/account_data_table.go @@ -51,13 +51,13 @@ const selectMaxAccountDataIDSQL = "" + type accountDataStatements struct { db *sql.DB - streamIDStatements *streamIDStatements + streamIDStatements *StreamIDStatements insertAccountDataStmt *sql.Stmt selectMaxAccountDataIDStmt *sql.Stmt selectAccountDataInRangeStmt *sql.Stmt } -func NewSqliteAccountDataTable(db *sql.DB, streamID *streamIDStatements) (tables.AccountData, error) { +func NewSqliteAccountDataTable(db *sql.DB, streamID *StreamIDStatements) (tables.AccountData, error) { s := &accountDataStatements{ db: db, streamIDStatements: streamID, diff --git a/syncapi/storage/sqlite3/backwards_extremities_table.go b/syncapi/storage/sqlite3/backwards_extremities_table.go index 662cb0252..c5674dded 100644 --- a/syncapi/storage/sqlite3/backwards_extremities_table.go +++ b/syncapi/storage/sqlite3/backwards_extremities_table.go @@ -47,15 +47,11 @@ const selectBackwardExtremitiesForRoomSQL = "" + const deleteBackwardExtremitySQL = "" + "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" -const deleteBackwardExtremitiesForRoomSQL = "" + - "DELETE FROM syncapi_backward_extremities WHERE room_id = $1" - type backwardExtremitiesStatements struct { db *sql.DB insertBackwardExtremityStmt *sql.Stmt selectBackwardExtremitiesForRoomStmt *sql.Stmt deleteBackwardExtremityStmt *sql.Stmt - deleteBackwardExtremitiesForRoomStmt *sql.Stmt } func NewSqliteBackwardsExtremitiesTable(db *sql.DB) (tables.BackwardsExtremities, error) { @@ -75,9 +71,6 @@ func NewSqliteBackwardsExtremitiesTable(db *sql.DB) (tables.BackwardsExtremities if s.deleteBackwardExtremityStmt, err = db.Prepare(deleteBackwardExtremitySQL); err != nil { return nil, err } - if s.deleteBackwardExtremitiesForRoomStmt, err = db.Prepare(deleteBackwardExtremitiesForRoomSQL); err != nil { - return nil, err - } return s, nil } @@ -116,10 +109,3 @@ func (s *backwardExtremitiesStatements) DeleteBackwardExtremity( _, err = sqlutil.TxStmt(txn, s.deleteBackwardExtremityStmt).ExecContext(ctx, roomID, knownEventID) return err } - -func (s *backwardExtremitiesStatements) DeleteBackwardExtremitiesForRoom( - ctx context.Context, txn *sql.Tx, roomID string, -) (err error) { - _, err = sqlutil.TxStmt(txn, s.deleteBackwardExtremitiesForRoomStmt).ExecContext(ctx, roomID) - return err -} diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index 473aa49b0..464f32e04 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -90,7 +90,7 @@ const selectEventsWithEventIDsSQL = "" + type currentRoomStateStatements struct { db *sql.DB - streamIDStatements *streamIDStatements + streamIDStatements *StreamIDStatements upsertRoomStateStmt *sql.Stmt deleteRoomStateByEventIDStmt *sql.Stmt deleteRoomStateForRoomStmt *sql.Stmt @@ -100,7 +100,7 @@ type currentRoomStateStatements struct { selectStateEventStmt *sql.Stmt } -func NewSqliteCurrentRoomStateTable(db *sql.DB, streamID *streamIDStatements) (tables.CurrentRoomState, error) { +func NewSqliteCurrentRoomStateTable(db *sql.DB, streamID *StreamIDStatements) (tables.CurrentRoomState, error) { s := ¤tRoomStateStatements{ db: db, streamIDStatements: streamID, diff --git a/syncapi/storage/sqlite3/ignores_table.go b/syncapi/storage/sqlite3/ignores_table.go new file mode 100644 index 000000000..f4afca55e --- /dev/null +++ b/syncapi/storage/sqlite3/ignores_table.go @@ -0,0 +1,87 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "encoding/json" + + "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/types" +) + +const ignoresSchema = ` +-- Stores data about ignoress +CREATE TABLE IF NOT EXISTS syncapi_ignores ( + -- The user ID whose ignore list this belongs to. + user_id TEXT NOT NULL, + ignores_json TEXT NOT NULL, + PRIMARY KEY(user_id) +); +` + +const selectIgnoresSQL = "" + + "SELECT ignores_json FROM syncapi_ignores WHERE user_id = $1" + +const upsertIgnoresSQL = "" + + "INSERT INTO syncapi_ignores (user_id, ignores_json) VALUES ($1, $2)" + + " ON CONFLICT DO UPDATE set ignores_json = $2" + +type ignoresStatements struct { + selectIgnoresStmt *sql.Stmt + upsertIgnoresStmt *sql.Stmt +} + +func NewSqliteIgnoresTable(db *sql.DB) (tables.Ignores, error) { + _, err := db.Exec(ignoresSchema) + if err != nil { + return nil, err + } + s := &ignoresStatements{} + if s.selectIgnoresStmt, err = db.Prepare(selectIgnoresSQL); err != nil { + return nil, err + } + if s.upsertIgnoresStmt, err = db.Prepare(upsertIgnoresSQL); err != nil { + return nil, err + } + return s, nil +} + +func (s *ignoresStatements) SelectIgnores( + ctx context.Context, userID string, +) (*types.IgnoredUsers, error) { + var ignoresData []byte + err := s.selectIgnoresStmt.QueryRowContext(ctx, userID).Scan(&ignoresData) + if err != nil { + return nil, err + } + var ignores types.IgnoredUsers + if err = json.Unmarshal(ignoresData, &ignores); err != nil { + return nil, err + } + return &ignores, nil +} + +func (s *ignoresStatements) UpsertIgnores( + ctx context.Context, userID string, ignores *types.IgnoredUsers, +) error { + ignoresJSON, err := json.Marshal(ignores) + if err != nil { + return err + } + _, err = s.upsertIgnoresStmt.ExecContext(ctx, userID, ignoresJSON) + return err +} diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go index 0a6823cc0..58ab8461e 100644 --- a/syncapi/storage/sqlite3/invites_table.go +++ b/syncapi/storage/sqlite3/invites_table.go @@ -59,14 +59,14 @@ const selectMaxInviteIDSQL = "" + type inviteEventsStatements struct { db *sql.DB - streamIDStatements *streamIDStatements + streamIDStatements *StreamIDStatements insertInviteEventStmt *sql.Stmt selectInviteEventsInRangeStmt *sql.Stmt deleteInviteEventStmt *sql.Stmt selectMaxInviteIDStmt *sql.Stmt } -func NewSqliteInvitesTable(db *sql.DB, streamID *streamIDStatements) (tables.Invites, error) { +func NewSqliteInvitesTable(db *sql.DB, streamID *StreamIDStatements) (tables.Invites, error) { s := &inviteEventsStatements{ db: db, streamIDStatements: streamID, diff --git a/syncapi/storage/sqlite3/memberships_table.go b/syncapi/storage/sqlite3/memberships_table.go index e5445e815..9f3530ccd 100644 --- a/syncapi/storage/sqlite3/memberships_table.go +++ b/syncapi/storage/sqlite3/memberships_table.go @@ -18,7 +18,6 @@ import ( "context" "database/sql" "fmt" - "strings" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" @@ -57,15 +56,15 @@ const upsertMembershipSQL = "" + " ON CONFLICT (room_id, user_id, membership)" + " DO UPDATE SET event_id = $4, stream_pos = $5, topological_pos = $6" -const selectMembershipSQL = "" + - "SELECT event_id, stream_pos, topological_pos FROM syncapi_memberships" + - " WHERE room_id = $1 AND user_id = $2 AND membership IN ($3)" + - " ORDER BY stream_pos DESC" + - " LIMIT 1" +const selectMembershipCountSQL = "" + + "SELECT COUNT(*) FROM (" + + " SELECT * FROM syncapi_memberships WHERE room_id = $1 AND stream_pos <= $2 GROUP BY user_id HAVING(max(stream_pos))" + + ") t WHERE t.membership = $3" type membershipsStatements struct { - db *sql.DB - upsertMembershipStmt *sql.Stmt + db *sql.DB + upsertMembershipStmt *sql.Stmt + selectMembershipCountStmt *sql.Stmt } func NewSqliteMembershipsTable(db *sql.DB) (tables.Memberships, error) { @@ -79,6 +78,9 @@ func NewSqliteMembershipsTable(db *sql.DB) (tables.Memberships, error) { if s.upsertMembershipStmt, err = db.Prepare(upsertMembershipSQL); err != nil { return nil, err } + if s.selectMembershipCountStmt, err = db.Prepare(selectMembershipCountSQL); err != nil { + return nil, err + } return s, nil } @@ -102,18 +104,10 @@ func (s *membershipsStatements) UpsertMembership( return err } -func (s *membershipsStatements) SelectMembership( - ctx context.Context, txn *sql.Tx, roomID, userID, memberships []string, -) (eventID string, streamPos, topologyPos types.StreamPosition, err error) { - params := []interface{}{roomID, userID} - for _, membership := range memberships { - params = append(params, membership) - } - orig := strings.Replace(selectMembershipSQL, "($3)", sqlutil.QueryVariadicOffset(len(memberships), 2), 1) - stmt, err := s.db.Prepare(orig) - if err != nil { - return "", 0, 0, err - } - err = sqlutil.TxStmt(txn, stmt).QueryRowContext(ctx, params...).Scan(&eventID, &streamPos, &topologyPos) +func (s *membershipsStatements) SelectMembershipCount( + ctx context.Context, txn *sql.Tx, roomID, membership string, pos types.StreamPosition, +) (count int, err error) { + stmt := sqlutil.TxStmt(txn, s.selectMembershipCountStmt) + err = stmt.QueryRowContext(ctx, roomID, pos, membership).Scan(&count) return } diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index acd959696..9da9d776e 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -58,7 +58,7 @@ const insertEventSQL = "" + "ON CONFLICT (event_id) DO UPDATE SET exclude_from_sync = (excluded.exclude_from_sync AND $13)" const selectEventsSQL = "" + - "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = $1" + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id IN ($1)" const selectRecentEventsSQL = "" + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + @@ -111,9 +111,8 @@ const selectContextAfterEventSQL = "" + type outputRoomEventsStatements struct { db *sql.DB - streamIDStatements *streamIDStatements + streamIDStatements *StreamIDStatements insertEventStmt *sql.Stmt - selectEventsStmt *sql.Stmt selectMaxEventIDStmt *sql.Stmt updateEventJSONStmt *sql.Stmt deleteEventsForRoomStmt *sql.Stmt @@ -122,7 +121,7 @@ type outputRoomEventsStatements struct { selectContextAfterEventStmt *sql.Stmt } -func NewSqliteEventsTable(db *sql.DB, streamID *streamIDStatements) (tables.Events, error) { +func NewSqliteEventsTable(db *sql.DB, streamID *StreamIDStatements) (tables.Events, error) { s := &outputRoomEventsStatements{ db: db, streamIDStatements: streamID, @@ -133,7 +132,6 @@ func NewSqliteEventsTable(db *sql.DB, streamID *streamIDStatements) (tables.Even } return s, sqlutil.StatementList{ {&s.insertEventStmt, insertEventSQL}, - {&s.selectEventsStmt, selectEventsSQL}, {&s.selectMaxEventIDStmt, selectMaxEventIDSQL}, {&s.updateEventJSONStmt, updateEventJSONSQL}, {&s.deleteEventsForRoomStmt, deleteEventsForRoomSQL}, @@ -421,21 +419,43 @@ func (s *outputRoomEventsStatements) SelectEarlyEvents( // selectEvents returns the events for the given event IDs. If an event is // missing from the database, it will be omitted. func (s *outputRoomEventsStatements) SelectEvents( - ctx context.Context, txn *sql.Tx, eventIDs []string, + ctx context.Context, txn *sql.Tx, eventIDs []string, preserveOrder bool, ) ([]types.StreamEvent, error) { - var returnEvents []types.StreamEvent - stmt := sqlutil.TxStmt(txn, s.selectEventsStmt) - for _, eventID := range eventIDs { - rows, err := stmt.QueryContext(ctx, eventID) - if err != nil { - return nil, err - } - if streamEvents, err := rowsToStreamEvents(rows); err == nil { - returnEvents = append(returnEvents, streamEvents...) - } - internal.CloseAndLogIfError(ctx, rows, "selectEvents: rows.close() failed") + iEventIDs := make([]interface{}, len(eventIDs)) + for i := range eventIDs { + iEventIDs[i] = eventIDs[i] } - return returnEvents, nil + selectSQL := strings.Replace(selectEventsSQL, "($1)", sqlutil.QueryVariadic(len(eventIDs)), 1) + var rows *sql.Rows + var err error + if txn != nil { + rows, err = txn.QueryContext(ctx, selectSQL, iEventIDs...) + } else { + rows, err = s.db.QueryContext(ctx, selectSQL, iEventIDs...) + } + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "selectEvents: rows.close() failed") + streamEvents, err := rowsToStreamEvents(rows) + if err != nil { + return nil, err + } + if preserveOrder { + var returnEvents []types.StreamEvent + eventMap := make(map[string]types.StreamEvent) + for _, ev := range streamEvents { + eventMap[ev.EventID()] = ev + } + for _, eventID := range eventIDs { + ev, ok := eventMap[eventID] + if ok { + returnEvents = append(returnEvents, ev) + } + } + return returnEvents, nil + } + return streamEvents, nil } func (s *outputRoomEventsStatements) DeleteEventsForRoom( diff --git a/syncapi/storage/sqlite3/output_room_events_topology_table.go b/syncapi/storage/sqlite3/output_room_events_topology_table.go index b972ae285..b2fb77417 100644 --- a/syncapi/storage/sqlite3/output_room_events_topology_table.go +++ b/syncapi/storage/sqlite3/output_room_events_topology_table.go @@ -78,7 +78,6 @@ type outputRoomEventsTopologyStatements struct { selectEventIDsInRangeDESCStmt *sql.Stmt selectPositionInTopologyStmt *sql.Stmt selectMaxPositionInTopologyStmt *sql.Stmt - deleteTopologyForRoomStmt *sql.Stmt selectStreamToTopologicalPositionAscStmt *sql.Stmt selectStreamToTopologicalPositionDescStmt *sql.Stmt } @@ -191,10 +190,3 @@ func (s *outputRoomEventsTopologyStatements) SelectMaxPositionInTopology( err = stmt.QueryRowContext(ctx, roomID).Scan(&pos, &spos) return } - -func (s *outputRoomEventsTopologyStatements) DeleteTopologyForRoom( - ctx context.Context, txn *sql.Tx, roomID string, -) (err error) { - _, err = sqlutil.TxStmt(txn, s.deleteTopologyForRoomStmt).ExecContext(ctx, roomID) - return err -} diff --git a/syncapi/storage/sqlite3/peeks_table.go b/syncapi/storage/sqlite3/peeks_table.go index c93c82051..5ee86448c 100644 --- a/syncapi/storage/sqlite3/peeks_table.go +++ b/syncapi/storage/sqlite3/peeks_table.go @@ -66,7 +66,7 @@ const selectMaxPeekIDSQL = "" + type peekStatements struct { db *sql.DB - streamIDStatements *streamIDStatements + streamIDStatements *StreamIDStatements insertPeekStmt *sql.Stmt deletePeekStmt *sql.Stmt deletePeeksStmt *sql.Stmt @@ -75,7 +75,7 @@ type peekStatements struct { selectMaxPeekIDStmt *sql.Stmt } -func NewSqlitePeeksTable(db *sql.DB, streamID *streamIDStatements) (tables.Peeks, error) { +func NewSqlitePeeksTable(db *sql.DB, streamID *StreamIDStatements) (tables.Peeks, error) { _, err := db.Exec(peeksSchema) if err != nil { return nil, err diff --git a/syncapi/storage/sqlite3/presence_table.go b/syncapi/storage/sqlite3/presence_table.go new file mode 100644 index 000000000..00b16458d --- /dev/null +++ b/syncapi/storage/sqlite3/presence_table.go @@ -0,0 +1,177 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/gomatrixserverlib" +) + +const presenceSchema = ` +-- Stores data about presence +CREATE TABLE IF NOT EXISTS syncapi_presence ( + -- The ID + id BIGINT NOT NULL, + -- The Matrix user ID + user_id TEXT NOT NULL, + -- The actual presence + presence INT NOT NULL, + -- The status message + status_msg TEXT, + -- The last time an action was received by this user + last_active_ts BIGINT NOT NULL, + CONSTRAINT presence_presences_unique UNIQUE (user_id) +); +CREATE INDEX IF NOT EXISTS syncapi_presence_user_id ON syncapi_presence(user_id); +` + +const upsertPresenceSQL = "" + + "INSERT INTO syncapi_presence AS p" + + " (id, user_id, presence, status_msg, last_active_ts)" + + " VALUES ($1, $2, $3, $4, $5)" + + " ON CONFLICT (user_id)" + + " DO UPDATE SET id = $6, " + + " presence = $7, status_msg = COALESCE($8, p.status_msg), last_active_ts = $9" + + " RETURNING id" + +const upsertPresenceFromSyncSQL = "" + + "INSERT INTO syncapi_presence AS p" + + " (id, user_id, presence, last_active_ts)" + + " VALUES ($1, $2, $3, $4)" + + " ON CONFLICT (user_id)" + + " DO UPDATE SET id = $5, " + + " presence = $6, last_active_ts = $7" + + " RETURNING id" + +const selectPresenceForUserSQL = "" + + "SELECT presence, status_msg, last_active_ts" + + " FROM syncapi_presence" + + " WHERE user_id = $1 LIMIT 1" + +const selectMaxPresenceSQL = "" + + "SELECT COALESCE(MAX(id), 0) FROM syncapi_presence" + +const selectPresenceAfter = "" + + " SELECT id, user_id, presence, status_msg, last_active_ts" + + " FROM syncapi_presence" + + " WHERE id > $1" + +type presenceStatements struct { + db *sql.DB + streamIDStatements *StreamIDStatements + upsertPresenceStmt *sql.Stmt + upsertPresenceFromSyncStmt *sql.Stmt + selectPresenceForUsersStmt *sql.Stmt + selectMaxPresenceStmt *sql.Stmt + selectPresenceAfterStmt *sql.Stmt +} + +func NewSqlitePresenceTable(db *sql.DB, streamID *StreamIDStatements) (*presenceStatements, error) { + _, err := db.Exec(presenceSchema) + if err != nil { + return nil, err + } + s := &presenceStatements{ + db: db, + streamIDStatements: streamID, + } + return s, sqlutil.StatementList{ + {&s.upsertPresenceStmt, upsertPresenceSQL}, + {&s.upsertPresenceFromSyncStmt, upsertPresenceFromSyncSQL}, + {&s.selectPresenceForUsersStmt, selectPresenceForUserSQL}, + {&s.selectMaxPresenceStmt, selectMaxPresenceSQL}, + {&s.selectPresenceAfterStmt, selectPresenceAfter}, + }.Prepare(db) +} + +// UpsertPresence creates/updates a presence status. +func (p *presenceStatements) UpsertPresence( + ctx context.Context, + txn *sql.Tx, + userID string, + statusMsg *string, + presence types.Presence, + lastActiveTS gomatrixserverlib.Timestamp, + fromSync bool, +) (pos types.StreamPosition, err error) { + pos, err = p.streamIDStatements.nextPresenceID(ctx, txn) + if err != nil { + return pos, err + } + + if fromSync { + stmt := sqlutil.TxStmt(txn, p.upsertPresenceFromSyncStmt) + err = stmt.QueryRowContext(ctx, + pos, userID, presence, + lastActiveTS, pos, + presence, lastActiveTS).Scan(&pos) + } else { + stmt := sqlutil.TxStmt(txn, p.upsertPresenceStmt) + err = stmt.QueryRowContext(ctx, + pos, userID, presence, + statusMsg, lastActiveTS, pos, + presence, statusMsg, lastActiveTS).Scan(&pos) + } + return +} + +// GetPresenceForUser returns the current presence of a user. +func (p *presenceStatements) GetPresenceForUser( + ctx context.Context, txn *sql.Tx, + userID string, +) (*types.PresenceInternal, error) { + result := &types.PresenceInternal{ + UserID: userID, + } + stmt := sqlutil.TxStmt(txn, p.selectPresenceForUsersStmt) + err := stmt.QueryRowContext(ctx, userID).Scan(&result.Presence, &result.ClientFields.StatusMsg, &result.LastActiveTS) + result.ClientFields.Presence = result.Presence.String() + return result, err +} + +func (p *presenceStatements) GetMaxPresenceID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { + stmt := sqlutil.TxStmt(txn, p.selectMaxPresenceStmt) + err = stmt.QueryRowContext(ctx).Scan(&pos) + return +} + +// GetPresenceAfter returns the changes presences after a given stream id +func (p *presenceStatements) GetPresenceAfter( + ctx context.Context, txn *sql.Tx, + after types.StreamPosition, +) (presences map[string]*types.PresenceInternal, err error) { + presences = make(map[string]*types.PresenceInternal) + stmt := sqlutil.TxStmt(txn, p.selectPresenceAfterStmt) + + rows, err := stmt.QueryContext(ctx, after) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "GetPresenceAfter: failed to close rows") + for rows.Next() { + qryRes := &types.PresenceInternal{} + if err := rows.Scan(&qryRes.StreamPos, &qryRes.UserID, &qryRes.Presence, &qryRes.ClientFields.StatusMsg, &qryRes.LastActiveTS); err != nil { + return nil, err + } + qryRes.ClientFields.Presence = qryRes.Presence.String() + presences[qryRes.UserID] = qryRes + } + return presences, rows.Err() +} diff --git a/syncapi/storage/sqlite3/receipt_table.go b/syncapi/storage/sqlite3/receipt_table.go index 9111a39f6..bd778bf3c 100644 --- a/syncapi/storage/sqlite3/receipt_table.go +++ b/syncapi/storage/sqlite3/receipt_table.go @@ -20,7 +20,6 @@ import ( "fmt" "strings" - "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" @@ -60,13 +59,13 @@ const selectMaxReceiptIDSQL = "" + type receiptStatements struct { db *sql.DB - streamIDStatements *streamIDStatements + streamIDStatements *StreamIDStatements upsertReceipt *sql.Stmt selectRoomReceipts *sql.Stmt selectMaxReceiptID *sql.Stmt } -func NewSqliteReceiptsTable(db *sql.DB, streamID *streamIDStatements) (tables.Receipts, error) { +func NewSqliteReceiptsTable(db *sql.DB, streamID *StreamIDStatements) (tables.Receipts, error) { _, err := db.Exec(receiptsSchema) if err != nil { return nil, err @@ -99,7 +98,7 @@ func (r *receiptStatements) UpsertReceipt(ctx context.Context, txn *sql.Tx, room } // SelectRoomReceiptsAfter select all receipts for a given room after a specific timestamp -func (r *receiptStatements) SelectRoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []api.OutputReceiptEvent, error) { +func (r *receiptStatements) SelectRoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []types.OutputReceiptEvent, error) { selectSQL := strings.Replace(selectRoomReceipts, "($2)", sqlutil.QueryVariadicOffset(len(roomIDs), 1), 1) var lastPos types.StreamPosition params := make([]interface{}, len(roomIDs)+1) @@ -112,9 +111,9 @@ func (r *receiptStatements) SelectRoomReceiptsAfter(ctx context.Context, roomIDs return 0, nil, fmt.Errorf("unable to query room receipts: %w", err) } defer internal.CloseAndLogIfError(ctx, rows, "SelectRoomReceiptsAfter: rows.close() failed") - var res []api.OutputReceiptEvent + var res []types.OutputReceiptEvent for rows.Next() { - r := api.OutputReceiptEvent{} + r := types.OutputReceiptEvent{} var id types.StreamPosition err = rows.Scan(&id, &r.RoomID, &r.Type, &r.UserID, &r.EventID, &r.Timestamp) if err != nil { diff --git a/syncapi/storage/sqlite3/stream_id_table.go b/syncapi/storage/sqlite3/stream_id_table.go index 2be3ae93d..71980b806 100644 --- a/syncapi/storage/sqlite3/stream_id_table.go +++ b/syncapi/storage/sqlite3/stream_id_table.go @@ -24,18 +24,20 @@ INSERT INTO syncapi_stream_id (stream_name, stream_id) VALUES ("accountdata", 0) ON CONFLICT DO NOTHING; INSERT INTO syncapi_stream_id (stream_name, stream_id) VALUES ("invite", 0) ON CONFLICT DO NOTHING; +INSERT INTO syncapi_stream_id (stream_name, stream_id) VALUES ("presence", 0) + ON CONFLICT DO NOTHING; ` const increaseStreamIDStmt = "" + "UPDATE syncapi_stream_id SET stream_id = stream_id + 1 WHERE stream_name = $1" + " RETURNING stream_id" -type streamIDStatements struct { +type StreamIDStatements struct { db *sql.DB increaseStreamIDStmt *sql.Stmt } -func (s *streamIDStatements) prepare(db *sql.DB) (err error) { +func (s *StreamIDStatements) Prepare(db *sql.DB) (err error) { s.db = db _, err = db.Exec(streamIDTableSchema) if err != nil { @@ -47,26 +49,32 @@ func (s *streamIDStatements) prepare(db *sql.DB) (err error) { return } -func (s *streamIDStatements) nextPDUID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { +func (s *StreamIDStatements) nextPDUID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { increaseStmt := sqlutil.TxStmt(txn, s.increaseStreamIDStmt) err = increaseStmt.QueryRowContext(ctx, "global").Scan(&pos) return } -func (s *streamIDStatements) nextReceiptID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { +func (s *StreamIDStatements) nextReceiptID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { increaseStmt := sqlutil.TxStmt(txn, s.increaseStreamIDStmt) err = increaseStmt.QueryRowContext(ctx, "receipt").Scan(&pos) return } -func (s *streamIDStatements) nextInviteID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { +func (s *StreamIDStatements) nextInviteID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { increaseStmt := sqlutil.TxStmt(txn, s.increaseStreamIDStmt) err = increaseStmt.QueryRowContext(ctx, "invite").Scan(&pos) return } -func (s *streamIDStatements) nextAccountDataID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { +func (s *StreamIDStatements) nextAccountDataID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { increaseStmt := sqlutil.TxStmt(txn, s.increaseStreamIDStmt) err = increaseStmt.QueryRowContext(ctx, "accountdata").Scan(&pos) return } + +func (s *StreamIDStatements) nextPresenceID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { + increaseStmt := sqlutil.TxStmt(txn, s.increaseStreamIDStmt) + err = increaseStmt.QueryRowContext(ctx, "presence").Scan(&pos) + return +} diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index cb7e3b46f..dfc289482 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -30,7 +30,7 @@ type SyncServerDatasource struct { shared.Database db *sql.DB writer sqlutil.Writer - streamID streamIDStatements + streamID StreamIDStatements } // NewDatabase creates a new sync server database @@ -49,7 +49,7 @@ func NewDatabase(dbProperties *config.DatabaseOptions) (*SyncServerDatasource, e } func (d *SyncServerDatasource) prepare(dbProperties *config.DatabaseOptions) (err error) { - if err = d.streamID.prepare(d.db); err != nil { + if err = d.streamID.Prepare(d.db); err != nil { return err } accountData, err := NewSqliteAccountDataTable(d.db, &d.streamID) @@ -100,6 +100,14 @@ func (d *SyncServerDatasource) prepare(dbProperties *config.DatabaseOptions) (er if err != nil { return err } + ignores, err := NewSqliteIgnoresTable(d.db) + if err != nil { + return err + } + presence, err := NewSqlitePresenceTable(d.db, &d.streamID) + if err != nil { + return err + } m := sqlutil.NewMigrations() deltas.LoadFixSequences(m) deltas.LoadRemoveSendToDeviceSentColumn(m) @@ -121,6 +129,8 @@ func (d *SyncServerDatasource) prepare(dbProperties *config.DatabaseOptions) (er Receipts: receipts, Memberships: memberships, NotificationData: notificationData, + Ignores: ignores, + Presence: presence, } return nil } diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index 864322001..4e1634ece 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -1,121 +1,29 @@ package storage_test -// TODO: Fix these tests -/* import ( "context" - "crypto/ed25519" - "encoding/json" "fmt" - "os" + "reflect" "testing" - "time" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/syncapi/storage" - "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" "github.com/matrix-org/dendrite/syncapi/types" - userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/test" "github.com/matrix-org/gomatrixserverlib" ) -var ( - ctx = context.Background() - emptyStateKey = "" - testOrigin = gomatrixserverlib.ServerName("hollow.knight") - testRoomID = fmt.Sprintf("!hallownest:%s", testOrigin) - testUserIDA = fmt.Sprintf("@hornet:%s", testOrigin) - testUserIDB = fmt.Sprintf("@paleking:%s", testOrigin) - testUserDeviceA = userapi.Device{ - UserID: testUserIDA, - ID: "device_id_A", - DisplayName: "Device A", - } - testRoomVersion = gomatrixserverlib.RoomVersionV4 - testKeyID = gomatrixserverlib.KeyID("ed25519:storage_test") - testPrivateKey = ed25519.NewKeyFromSeed([]byte{ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, - 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, - }) -) +var ctx = context.Background() -func MustCreateEvent(t *testing.T, roomID string, prevs []*gomatrixserverlib.HeaderedEvent, b *gomatrixserverlib.EventBuilder) *gomatrixserverlib.HeaderedEvent { - b.RoomID = roomID - if prevs != nil { - prevIDs := make([]string, len(prevs)) - for i := range prevs { - prevIDs[i] = prevs[i].EventID() - } - b.PrevEvents = prevIDs - } - e, err := b.Build(time.Now(), testOrigin, testKeyID, testPrivateKey, testRoomVersion) - if err != nil { - t.Fatalf("failed to build event: %s", err) - } - return e.Headered(testRoomVersion) -} - -func MustCreateDatabase(t *testing.T) storage.Database { - dbname := fmt.Sprintf("test_%s.db", t.Name()) - if _, err := os.Stat(dbname); err == nil { - if err = os.Remove(dbname); err != nil { - t.Fatalf("tried to delete stale test database but failed: %s", err) - } - } - db, err := sqlite3.NewDatabase(&config.DatabaseOptions{ - ConnectionString: config.DataSource(fmt.Sprintf("file:%s", dbname)), +func MustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) { + connStr, close := test.PrepareDBConnectionString(t, dbType) + db, err := storage.NewSyncServerDatasource(&config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), }) if err != nil { t.Fatalf("NewSyncServerDatasource returned %s", err) } - return db -} - -// Create a list of events which include a create event, join event and some messages. -func SimpleRoom(t *testing.T, roomID, userA, userB string) (msgs []*gomatrixserverlib.HeaderedEvent, state []*gomatrixserverlib.HeaderedEvent) { - var events []*gomatrixserverlib.HeaderedEvent - events = append(events, MustCreateEvent(t, roomID, nil, &gomatrixserverlib.EventBuilder{ - Content: []byte(fmt.Sprintf(`{"room_version":"4","creator":"%s"}`, userA)), - Type: "m.room.create", - StateKey: &emptyStateKey, - Sender: userA, - Depth: int64(len(events) + 1), - })) - state = append(state, events[len(events)-1]) - events = append(events, MustCreateEvent(t, roomID, []*gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ - Content: []byte(`{"membership":"join"}`), - Type: "m.room.member", - StateKey: &userA, - Sender: userA, - Depth: int64(len(events) + 1), - })) - state = append(state, events[len(events)-1]) - for i := 0; i < 10; i++ { - events = append(events, MustCreateEvent(t, roomID, []*gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ - Content: []byte(fmt.Sprintf(`{"body":"Message A %d"}`, i+1)), - Type: "m.room.message", - Sender: userA, - Depth: int64(len(events) + 1), - })) - } - events = append(events, MustCreateEvent(t, roomID, []*gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ - Content: []byte(`{"membership":"join"}`), - Type: "m.room.member", - StateKey: &userB, - Sender: userB, - Depth: int64(len(events) + 1), - })) - state = append(state, events[len(events)-1]) - for i := 0; i < 10; i++ { - events = append(events, MustCreateEvent(t, roomID, []*gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ - Content: []byte(fmt.Sprintf(`{"body":"Message B %d"}`, i+1)), - Type: "m.room.message", - Sender: userB, - Depth: int64(len(events) + 1), - })) - } - - return events, state + return db, close } func MustWriteEvents(t *testing.T, db storage.Database, events []*gomatrixserverlib.HeaderedEvent) (positions []types.StreamPosition) { @@ -131,206 +39,157 @@ func MustWriteEvents(t *testing.T, db storage.Database, events []*gomatrixserver if err != nil { t.Fatalf("WriteEvent failed: %s", err) } - fmt.Println("Event ID", ev.EventID(), " spos=", pos, "depth=", ev.Depth()) + t.Logf("Event ID %s spos=%v depth=%v", ev.EventID(), pos, ev.Depth()) positions = append(positions, pos) } return } func TestWriteEvents(t *testing.T) { - t.Parallel() - db := MustCreateDatabase(t) - events, _ := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) - MustWriteEvents(t, db, events) + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + alice := test.NewUser() + r := test.NewRoom(t, alice) + db, close := MustCreateDatabase(t, dbType) + defer close() + MustWriteEvents(t, db, r.Events()) + }) } -// These tests assert basic functionality of the IncrementalSync and CompleteSync functions. -func TestSyncResponse(t *testing.T) { - t.Parallel() - db := MustCreateDatabase(t) - events, state := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) - positions := MustWriteEvents(t, db, events) - latest, err := db.SyncPosition(ctx) - if err != nil { - t.Fatalf("failed to get SyncPosition: %s", err) - } +// These tests assert basic functionality of RecentEvents for PDUs +func TestRecentEventsPDU(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, close := MustCreateDatabase(t, dbType) + defer close() + alice := test.NewUser() + // dummy room to make sure SQL queries are filtering on room ID + MustWriteEvents(t, db, test.NewRoom(t, alice).Events()) - testCases := []struct { - Name string - DoSync func() (*types.Response, error) - WantTimeline []*gomatrixserverlib.HeaderedEvent - WantState []*gomatrixserverlib.HeaderedEvent - }{ - // The purpose of this test is to make sure that incremental syncs are including up to the latest events. - // It's a basic sanity test that sync works. It creates a `since` token that is on the penultimate event. - // It makes sure the response includes the final event. - { - Name: "IncrementalSync penultimate", - DoSync: func() (*types.Response, error) { - from := types.StreamingToken{ // pretend we are at the penultimate event - PDUPosition: positions[len(positions)-2], + // actual test room + r := test.NewRoom(t, alice) + r.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "hi"}) + events := r.Events() + positions := MustWriteEvents(t, db, events) + + // dummy room to make sure SQL queries are filtering on room ID + MustWriteEvents(t, db, test.NewRoom(t, alice).Events()) + + latest, err := db.MaxStreamPositionForPDUs(ctx) + if err != nil { + t.Fatalf("failed to get MaxStreamPositionForPDUs: %s", err) + } + + testCases := []struct { + Name string + From types.StreamPosition + To types.StreamPosition + Limit int + ReverseOrder bool + WantEvents []*gomatrixserverlib.HeaderedEvent + WantLimited bool + }{ + // The purpose of this test is to make sure that incremental syncs are including up to the latest events. + // It's a basic sanity test that sync works. It creates a streaming position that is on the penultimate event. + // It makes sure the response includes the final event. + { + Name: "penultimate", + From: positions[len(positions)-2], // pretend we are at the penultimate event + To: latest, + Limit: 100, + WantEvents: events[len(events)-1:], + WantLimited: false, + }, + // The purpose of this test is to check that limits can be applied and work. + // This is critical for big rooms hence the test here. + { + Name: "limited", + From: 0, + To: latest, + Limit: 1, + WantEvents: events[len(events)-1:], + WantLimited: true, + }, + // The purpose of this test is to check that we can return every event with a high + // enough limit + { + Name: "large limited", + From: 0, + To: latest, + Limit: 100, + WantEvents: events, + WantLimited: false, + }, + // The purpose of this test is to check that we can return events in reverse order + { + Name: "reverse", + From: positions[len(positions)-3], // 2 events back + To: latest, + Limit: 100, + ReverseOrder: true, + WantEvents: test.Reversed(events[len(events)-2:]), + WantLimited: false, + }, + } + + for i := range testCases { + tc := testCases[i] + t.Run(tc.Name, func(st *testing.T) { + var filter gomatrixserverlib.RoomEventFilter + filter.Limit = tc.Limit + gotEvents, limited, err := db.RecentEvents(ctx, r.ID, types.Range{ + From: tc.From, + To: tc.To, + }, &filter, !tc.ReverseOrder, true) + if err != nil { + st.Fatalf("failed to do sync: %s", err) } - res := types.NewResponse() - return db.IncrementalSync(ctx, res, testUserDeviceA, from, latest, 5, false) - }, - WantTimeline: events[len(events)-1:], - }, - // The purpose of this test is to check that passing a `numRecentEventsPerRoom` correctly limits the - // number of returned events. This is critical for big rooms hence the test here. - { - Name: "IncrementalSync limited", - DoSync: func() (*types.Response, error) { - from := types.StreamingToken{ // pretend we are 10 events behind - PDUPosition: positions[len(positions)-11], + if limited != tc.WantLimited { + st.Errorf("got limited=%v want %v", limited, tc.WantLimited) } - res := types.NewResponse() - // limit is set to 5 - return db.IncrementalSync(ctx, res, testUserDeviceA, from, latest, 5, false) - }, - // want the last 5 events, NOT the last 10. - WantTimeline: events[len(events)-5:], - }, - // The purpose of this test is to check that CompleteSync returns all the current state as well as - // honouring the `numRecentEventsPerRoom` value - { - Name: "CompleteSync limited", - DoSync: func() (*types.Response, error) { - res := types.NewResponse() - // limit set to 5 - return db.CompleteSync(ctx, res, testUserDeviceA, 5) - }, - // want the last 5 events - WantTimeline: events[len(events)-5:], - // want all state for the room - WantState: state, - }, - // The purpose of this test is to check that CompleteSync can return everything with a high enough - // `numRecentEventsPerRoom`. - { - Name: "CompleteSync", - DoSync: func() (*types.Response, error) { - res := types.NewResponse() - return db.CompleteSync(ctx, res, testUserDeviceA, len(events)+1) - }, - WantTimeline: events, - // We want no state at all as that field in /sync is the delta between the token (beginning of time) - // and the START of the timeline. - }, - } - - for _, tc := range testCases { - t.Run(tc.Name, func(st *testing.T) { - res, err := tc.DoSync() - if err != nil { - st.Fatalf("failed to do sync: %s", err) - } - next := types.StreamingToken{ - PDUPosition: latest.PDUPosition, - TypingPosition: latest.TypingPosition, - ReceiptPosition: latest.ReceiptPosition, - SendToDevicePosition: latest.SendToDevicePosition, - } - if res.NextBatch.String() != next.String() { - st.Errorf("NextBatch got %s want %s", res.NextBatch, next.String()) - } - roomRes, ok := res.Rooms.Join[testRoomID] - if !ok { - st.Fatalf("IncrementalSync response missing room %s - response: %+v", testRoomID, res) - } - assertEventsEqual(st, "state for "+testRoomID, false, roomRes.State.Events, tc.WantState) - assertEventsEqual(st, "timeline for "+testRoomID, false, roomRes.Timeline.Events, tc.WantTimeline) - }) - } -} - -func TestGetEventsInRangeWithPrevBatch(t *testing.T) { - t.Parallel() - db := MustCreateDatabase(t) - events, _ := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) - positions := MustWriteEvents(t, db, events) - latest, err := db.SyncPosition(ctx) - if err != nil { - t.Fatalf("failed to get SyncPosition: %s", err) - } - from := types.StreamingToken{ - PDUPosition: positions[len(positions)-2], - } - - res := types.NewResponse() - res, err = db.IncrementalSync(ctx, res, testUserDeviceA, from, latest, 5, false) - if err != nil { - t.Fatalf("failed to IncrementalSync with latest token") - } - roomRes, ok := res.Rooms.Join[testRoomID] - if !ok { - t.Fatalf("IncrementalSync response missing room %s - response: %+v", testRoomID, res) - } - // returns the last event "Message 10" - assertEventsEqual(t, "IncrementalSync Timeline", false, roomRes.Timeline.Events, reversed(events[len(events)-1:])) - - prev := roomRes.Timeline.PrevBatch.String() - if prev == "" { - t.Fatalf("IncrementalSync expected prev_batch token") - } - prevBatchToken, err := types.NewTopologyTokenFromString(prev) - if err != nil { - t.Fatalf("failed to NewTopologyTokenFromString : %s", err) - } - // backpaginate 5 messages starting at the latest position. - // head towards the beginning of time - to := types.TopologyToken{} - paginatedEvents, err := db.GetEventsInTopologicalRange(ctx, &prevBatchToken, &to, testRoomID, 5, true) - if err != nil { - t.Fatalf("GetEventsInRange returned an error: %s", err) - } - gots := gomatrixserverlib.HeaderedToClientEvents(db.StreamEventsToEvents(&testUserDeviceA, paginatedEvents), gomatrixserverlib.FormatAll) - assertEventsEqual(t, "", true, gots, reversed(events[len(events)-6:len(events)-1])) -} - -// The purpose of this test is to ensure that backfill does indeed go backwards, using a stream token. -func TestGetEventsInRangeWithStreamToken(t *testing.T) { - t.Parallel() - db := MustCreateDatabase(t) - events, _ := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) - MustWriteEvents(t, db, events) - latest, err := db.SyncPosition(ctx) - if err != nil { - t.Fatalf("failed to get SyncPosition: %s", err) - } - // head towards the beginning of time - to := types.StreamingToken{} - - // backpaginate 5 messages starting at the latest position. - paginatedEvents, err := db.GetEventsInStreamingRange(ctx, &latest, &to, testRoomID, 5, true) - if err != nil { - t.Fatalf("GetEventsInRange returned an error: %s", err) - } - gots := gomatrixserverlib.HeaderedToClientEvents(db.StreamEventsToEvents(&testUserDeviceA, paginatedEvents), gomatrixserverlib.FormatAll) - assertEventsEqual(t, "", true, gots, reversed(events[len(events)-5:])) + if len(gotEvents) != len(tc.WantEvents) { + st.Errorf("got %d events, want %d", len(gotEvents), len(tc.WantEvents)) + } + for j := range gotEvents { + if !reflect.DeepEqual(gotEvents[j].JSON(), tc.WantEvents[j].JSON()) { + st.Errorf("event %d got %s want %s", j, string(gotEvents[j].JSON()), string(tc.WantEvents[j].JSON())) + } + } + }) + } + }) } // The purpose of this test is to ensure that backfill does indeed go backwards, using a topology token func TestGetEventsInRangeWithTopologyToken(t *testing.T) { - t.Parallel() - db := MustCreateDatabase(t) - events, _ := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) - MustWriteEvents(t, db, events) - from, err := db.MaxTopologicalPosition(ctx, testRoomID) - if err != nil { - t.Fatalf("failed to get MaxTopologicalPosition: %s", err) - } - // head towards the beginning of time - to := types.TopologyToken{} + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, close := MustCreateDatabase(t, dbType) + defer close() + alice := test.NewUser() + r := test.NewRoom(t, alice) + for i := 0; i < 10; i++ { + r.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": fmt.Sprintf("hi %d", i)}) + } + events := r.Events() + _ = MustWriteEvents(t, db, events) - // backpaginate 5 messages starting at the latest position. - paginatedEvents, err := db.GetEventsInTopologicalRange(ctx, &from, &to, testRoomID, 5, true) - if err != nil { - t.Fatalf("GetEventsInRange returned an error: %s", err) - } - gots := gomatrixserverlib.HeaderedToClientEvents(db.StreamEventsToEvents(&testUserDeviceA, paginatedEvents), gomatrixserverlib.FormatAll) - assertEventsEqual(t, "", true, gots, reversed(events[len(events)-5:])) + from, err := db.MaxTopologicalPosition(ctx, r.ID) + if err != nil { + t.Fatalf("failed to get MaxTopologicalPosition: %s", err) + } + t.Logf("max topo pos = %+v", from) + // head towards the beginning of time + to := types.TopologyToken{} + + // backpaginate 5 messages starting at the latest position. + paginatedEvents, err := db.GetEventsInTopologicalRange(ctx, &from, &to, r.ID, 5, true) + if err != nil { + t.Fatalf("GetEventsInTopologicalRange returned an error: %s", err) + } + gots := db.StreamEventsToEvents(nil, paginatedEvents) + test.AssertEventsEqual(t, gots, test.Reversed(events[len(events)-5:])) + }) } +/* // The purpose of this test is to make sure that backpagination returns all events, even if some events have the same depth. // For cases where events have the same depth, the streaming token should be used to tie break so events written via WriteEvent // will appear FIRST when going backwards. This test creates a DAG like: @@ -740,12 +599,4 @@ func topologyTokenBefore(t *testing.T, db storage.Database, eventID string) *typ tok.Decrement() return &tok } - -func reversed(in []*gomatrixserverlib.HeaderedEvent) []*gomatrixserverlib.HeaderedEvent { - out := make([]*gomatrixserverlib.HeaderedEvent, len(in)) - for i := 0; i < len(in); i++ { - out[i] = in[len(in)-i-1] - } - return out -} */ diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index 640b7dc31..a7df70248 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -18,7 +18,6 @@ import ( "context" "database/sql" - eduAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/types" @@ -60,7 +59,7 @@ type Events interface { SelectRecentEvents(ctx context.Context, txn *sql.Tx, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error) // SelectEarlyEvents returns the earliest events in the given room. SelectEarlyEvents(ctx context.Context, txn *sql.Tx, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter) ([]types.StreamEvent, error) - SelectEvents(ctx context.Context, txn *sql.Tx, eventIDs []string) ([]types.StreamEvent, error) + SelectEvents(ctx context.Context, txn *sql.Tx, eventIDs []string, preserveOrder bool) ([]types.StreamEvent, error) UpdateEventJSON(ctx context.Context, event *gomatrixserverlib.HeaderedEvent) error // DeleteEventsForRoom removes all event information for a room. This should only be done when removing the room entirely. DeleteEventsForRoom(ctx context.Context, txn *sql.Tx, roomID string) (err error) @@ -85,8 +84,6 @@ type Topology interface { SelectPositionInTopology(ctx context.Context, txn *sql.Tx, eventID string) (depth, spos types.StreamPosition, err error) // SelectMaxPositionInTopology returns the event which has the highest depth, and if there are multiple, the event with the highest stream position. SelectMaxPositionInTopology(ctx context.Context, txn *sql.Tx, roomID string) (depth types.StreamPosition, spos types.StreamPosition, err error) - // DeleteTopologyForRoom removes all topological information for a room. This should only be done when removing the room entirely. - DeleteTopologyForRoom(ctx context.Context, txn *sql.Tx, roomID string) (err error) // SelectStreamToTopologicalPosition converts a stream position to a topological position by finding the nearest topological position in the room. SelectStreamToTopologicalPosition(ctx context.Context, txn *sql.Tx, roomID string, streamPos types.StreamPosition, forward bool) (topoPos types.StreamPosition, err error) } @@ -133,8 +130,6 @@ type BackwardsExtremities interface { SelectBackwardExtremitiesForRoom(ctx context.Context, roomID string) (bwExtrems map[string][]string, err error) // DeleteBackwardExtremity removes a backwards extremity for a room, if one existed. DeleteBackwardExtremity(ctx context.Context, txn *sql.Tx, roomID, knownEventID string) (err error) - // DeleteBackwardExtremitiesFoorRoomID removes all backward extremities for a room. This should only be done when removing the room entirely. - DeleteBackwardExtremitiesForRoom(ctx context.Context, txn *sql.Tx, roomID string) (err error) } // SendToDevice tracks send-to-device messages which are sent to individual @@ -168,13 +163,13 @@ type Filter interface { type Receipts interface { UpsertReceipt(ctx context.Context, txn *sql.Tx, roomId, receiptType, userId, eventId string, timestamp gomatrixserverlib.Timestamp) (pos types.StreamPosition, err error) - SelectRoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []eduAPI.OutputReceiptEvent, error) + SelectRoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []types.OutputReceiptEvent, error) SelectMaxReceiptID(ctx context.Context, txn *sql.Tx) (id int64, err error) } type Memberships interface { UpsertMembership(ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, streamPos, topologicalPos types.StreamPosition) error - SelectMembership(ctx context.Context, txn *sql.Tx, roomID, userID, memberships []string) (eventID string, streamPos, topologyPos types.StreamPosition, err error) + SelectMembershipCount(ctx context.Context, txn *sql.Tx, roomID, membership string, pos types.StreamPosition) (count int, err error) } type NotificationData interface { @@ -182,3 +177,15 @@ type NotificationData interface { SelectUserUnreadCounts(ctx context.Context, userID string, fromExcl, toIncl types.StreamPosition) (map[string]*eventutil.NotificationData, error) SelectMaxID(ctx context.Context) (int64, error) } + +type Ignores interface { + SelectIgnores(ctx context.Context, userID string) (*types.IgnoredUsers, error) + UpsertIgnores(ctx context.Context, userID string, ignores *types.IgnoredUsers) error +} + +type Presence interface { + UpsertPresence(ctx context.Context, txn *sql.Tx, userID string, statusMsg *string, presence types.Presence, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (pos types.StreamPosition, err error) + GetPresenceForUser(ctx context.Context, txn *sql.Tx, userID string) (presence *types.PresenceInternal, err error) + GetMaxPresenceID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) + GetPresenceAfter(ctx context.Context, txn *sql.Tx, after types.StreamPosition) (presences map[string]*types.PresenceInternal, err error) +} diff --git a/syncapi/storage/tables/output_room_events_test.go b/syncapi/storage/tables/output_room_events_test.go new file mode 100644 index 000000000..7a81ffcd2 --- /dev/null +++ b/syncapi/storage/tables/output_room_events_test.go @@ -0,0 +1,82 @@ +package tables_test + +import ( + "context" + "database/sql" + "fmt" + "reflect" + "testing" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/syncapi/storage/postgres" + "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" + "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/test" +) + +func newOutputRoomEventsTable(t *testing.T, dbType test.DBType) (tables.Events, *sql.DB, func()) { + t.Helper() + connStr, close := test.PrepareDBConnectionString(t, dbType) + db, err := sqlutil.Open(&config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + }) + if err != nil { + t.Fatalf("failed to open db: %s", err) + } + + var tab tables.Events + switch dbType { + case test.DBTypePostgres: + tab, err = postgres.NewPostgresEventsTable(db) + case test.DBTypeSQLite: + var stream sqlite3.StreamIDStatements + if err = stream.Prepare(db); err != nil { + t.Fatalf("failed to prepare stream stmts: %s", err) + } + tab, err = sqlite3.NewSqliteEventsTable(db, &stream) + } + if err != nil { + t.Fatalf("failed to make new table: %s", err) + } + return tab, db, close +} + +func TestOutputRoomEventsTable(t *testing.T) { + ctx := context.Background() + alice := test.NewUser() + room := test.NewRoom(t, alice) + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + tab, db, close := newOutputRoomEventsTable(t, dbType) + defer close() + events := room.Events() + err := sqlutil.WithTransaction(db, func(txn *sql.Tx) error { + for _, ev := range events { + _, err := tab.InsertEvent(ctx, txn, ev, nil, nil, nil, false) + if err != nil { + return fmt.Errorf("failed to InsertEvent: %s", err) + } + } + // order = 2,0,3,1 + wantEventIDs := []string{ + events[2].EventID(), events[0].EventID(), events[3].EventID(), events[1].EventID(), + } + gotEvents, err := tab.SelectEvents(ctx, txn, wantEventIDs, true) + if err != nil { + return fmt.Errorf("failed to SelectEvents: %s", err) + } + gotEventIDs := make([]string, len(gotEvents)) + for i := range gotEvents { + gotEventIDs[i] = gotEvents[i].EventID() + } + if !reflect.DeepEqual(gotEventIDs, wantEventIDs) { + return fmt.Errorf("SelectEvents\ngot %v\n want %v", gotEventIDs, wantEventIDs) + } + + return nil + }) + if err != nil { + t.Fatalf("err: %s", err) + } + }) +} diff --git a/syncapi/storage/tables/topology_test.go b/syncapi/storage/tables/topology_test.go new file mode 100644 index 000000000..b6ece0b0d --- /dev/null +++ b/syncapi/storage/tables/topology_test.go @@ -0,0 +1,91 @@ +package tables_test + +import ( + "context" + "database/sql" + "fmt" + "testing" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/syncapi/storage/postgres" + "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" + "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/dendrite/test" +) + +func newTopologyTable(t *testing.T, dbType test.DBType) (tables.Topology, *sql.DB, func()) { + t.Helper() + connStr, close := test.PrepareDBConnectionString(t, dbType) + db, err := sqlutil.Open(&config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + }) + if err != nil { + t.Fatalf("failed to open db: %s", err) + } + + var tab tables.Topology + switch dbType { + case test.DBTypePostgres: + tab, err = postgres.NewPostgresTopologyTable(db) + case test.DBTypeSQLite: + tab, err = sqlite3.NewSqliteTopologyTable(db) + } + if err != nil { + t.Fatalf("failed to make new table: %s", err) + } + return tab, db, close +} + +func TestTopologyTable(t *testing.T) { + ctx := context.Background() + alice := test.NewUser() + room := test.NewRoom(t, alice) + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + tab, db, close := newTopologyTable(t, dbType) + defer close() + events := room.Events() + err := sqlutil.WithTransaction(db, func(txn *sql.Tx) error { + var highestPos types.StreamPosition + for i, ev := range events { + topoPos, err := tab.InsertEventInTopology(ctx, txn, ev, types.StreamPosition(i)) + if err != nil { + return fmt.Errorf("failed to InsertEventInTopology: %s", err) + } + // topo pos = depth, depth starts at 1, hence 1+i + if topoPos != types.StreamPosition(1+i) { + return fmt.Errorf("got topo pos %d want %d", topoPos, 1+i) + } + highestPos = topoPos + 1 + } + // check ordering works without limit + eventIDs, err := tab.SelectEventIDsInRange(ctx, txn, room.ID, 0, highestPos, highestPos, 100, true) + if err != nil { + return fmt.Errorf("failed to SelectEventIDsInRange: %s", err) + } + test.AssertEventIDsEqual(t, eventIDs, events[:]) + eventIDs, err = tab.SelectEventIDsInRange(ctx, txn, room.ID, 0, highestPos, highestPos, 100, false) + if err != nil { + return fmt.Errorf("failed to SelectEventIDsInRange: %s", err) + } + test.AssertEventIDsEqual(t, eventIDs, test.Reversed(events[:])) + // check ordering works with limit + eventIDs, err = tab.SelectEventIDsInRange(ctx, txn, room.ID, 0, highestPos, highestPos, 3, true) + if err != nil { + return fmt.Errorf("failed to SelectEventIDsInRange: %s", err) + } + test.AssertEventIDsEqual(t, eventIDs, events[:3]) + eventIDs, err = tab.SelectEventIDsInRange(ctx, txn, room.ID, 0, highestPos, highestPos, 3, false) + if err != nil { + return fmt.Errorf("failed to SelectEventIDsInRange: %s", err) + } + test.AssertEventIDsEqual(t, eventIDs, test.Reversed(events[len(events)-3:])) + + return nil + }) + if err != nil { + t.Fatalf("err: %s", err) + } + }) +} diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go index 70374c6a7..ddac9be2c 100644 --- a/syncapi/streams/stream_invite.go +++ b/syncapi/streams/stream_invite.go @@ -54,6 +54,10 @@ func (p *InviteStreamProvider) IncrementalSync( } for roomID, inviteEvent := range invites { + // skip ignored user events + if _, ok := req.IgnoredUsers.List[inviteEvent.Sender()]; ok { + continue + } ir := types.NewInviteResponse(inviteEvent) req.Response.Rooms.Invite[roomID] = *ir } diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index ccdac0864..ab200e007 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -2,6 +2,7 @@ package streams import ( "context" + "database/sql" "sync" "time" @@ -25,6 +26,7 @@ type PDUStreamProvider struct { tasks chan func() workers atomic.Int32 + userAPI userapi.UserInternalAPI } func (p *PDUStreamProvider) worker() { @@ -87,6 +89,10 @@ func (p *PDUStreamProvider) CompleteSync( stateFilter := req.Filter.Room.State eventFilter := req.Filter.Room.Timeline + if err = p.addIgnoredUsersToFilter(ctx, req, &eventFilter); err != nil { + req.Log.WithError(err).Error("unable to update event filter with ignored users") + } + // Build up a /sync response. Add joined rooms. var reqMutex sync.Mutex var reqWaitGroup sync.WaitGroup @@ -175,6 +181,10 @@ func (p *PDUStreamProvider) IncrementalSync( return to } + if err = p.addIgnoredUsersToFilter(ctx, req, &eventFilter); err != nil { + req.Log.WithError(err).Error("unable to update event filter with ignored users") + } + newPos = from for _, delta := range stateDeltas { var pos types.StreamPosition @@ -253,9 +263,25 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( updateLatestPosition(delta.StateEvents[len(delta.StateEvents)-1].EventID()) } + hasMembershipChange := false + for _, recentEvent := range recentStreamEvents { + if recentEvent.Type() == gomatrixserverlib.MRoomMember && recentEvent.StateKey() != nil { + hasMembershipChange = true + break + } + } + + // Work out how many members are in the room. + joinedCount, _ := p.DB.MembershipCount(ctx, delta.RoomID, gomatrixserverlib.Join, latestPosition) + invitedCount, _ := p.DB.MembershipCount(ctx, delta.RoomID, gomatrixserverlib.Invite, latestPosition) + switch delta.Membership { case gomatrixserverlib.Join: jr := types.NewJoinResponse() + if hasMembershipChange { + jr.Summary.JoinedMemberCount = &joinedCount + jr.Summary.InvitedMemberCount = &invitedCount + } jr.Timeline.PrevBatch = &prevBatch jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = limited @@ -367,12 +393,18 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( prevBatch.Decrement() } + // Work out how many members are in the room. + joinedCount, _ := p.DB.MembershipCount(ctx, roomID, gomatrixserverlib.Join, r.From) + invitedCount, _ := p.DB.MembershipCount(ctx, roomID, gomatrixserverlib.Invite, r.From) + // We don't include a device here as we don't need to send down // transaction IDs for complete syncs, but we do it anyway because Sytest demands it for: // "Can sync a room with a message with a transaction id" - which does a complete sync to check. recentEvents := p.DB.StreamEventsToEvents(device, recentStreamEvents) stateEvents = removeDuplicates(stateEvents, recentEvents) jr = types.NewJoinResponse() + jr.Summary.JoinedMemberCount = &joinedCount + jr.Summary.InvitedMemberCount = &invitedCount jr.Timeline.PrevBatch = prevBatch jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = limited @@ -380,6 +412,23 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( return jr, nil } +// addIgnoredUsersToFilter adds ignored users to the eventfilter and +// the syncreq itself for further use in streams. +func (p *PDUStreamProvider) addIgnoredUsersToFilter(ctx context.Context, req *types.SyncRequest, eventFilter *gomatrixserverlib.RoomEventFilter) error { + ignores, err := p.DB.IgnoresForUser(ctx, req.Device.UserID) + if err != nil { + if err == sql.ErrNoRows { + return nil + } + return err + } + req.IgnoredUsers = *ignores + for userID := range ignores.List { + eventFilter.NotSenders = append(eventFilter.NotSenders, userID) + } + return nil +} + func removeDuplicates(stateEvents, recentEvents []*gomatrixserverlib.HeaderedEvent) []*gomatrixserverlib.HeaderedEvent { for _, recentEv := range recentEvents { if recentEv.StateKey() == nil { diff --git a/syncapi/streams/stream_presence.go b/syncapi/streams/stream_presence.go new file mode 100644 index 000000000..9a6c5c130 --- /dev/null +++ b/syncapi/streams/stream_presence.go @@ -0,0 +1,171 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package streams + +import ( + "context" + "database/sql" + "encoding/json" + "sync" + + "github.com/matrix-org/dendrite/syncapi/notifier" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/gomatrixserverlib" +) + +type PresenceStreamProvider struct { + StreamProvider + // cache contains previously sent presence updates to avoid unneeded updates + cache sync.Map + notifier *notifier.Notifier +} + +func (p *PresenceStreamProvider) Setup() { + p.StreamProvider.Setup() + + id, err := p.DB.MaxStreamPositionForPresence(context.Background()) + if err != nil { + panic(err) + } + p.latest = id +} + +func (p *PresenceStreamProvider) CompleteSync( + ctx context.Context, + req *types.SyncRequest, +) types.StreamPosition { + return p.IncrementalSync(ctx, req, 0, p.LatestPosition(ctx)) +} + +func (p *PresenceStreamProvider) IncrementalSync( + ctx context.Context, + req *types.SyncRequest, + from, to types.StreamPosition, +) types.StreamPosition { + presences, err := p.DB.PresenceAfter(ctx, from) + if err != nil { + req.Log.WithError(err).Error("p.DB.PresenceAfter failed") + return from + } + + if len(presences) == 0 { + return to + } + + // add newly joined rooms user presences + newlyJoined := joinedRooms(req.Response, req.Device.UserID) + if len(newlyJoined) > 0 { + // TODO: This refreshes all lists and is quite expensive + // The notifier should update the lists itself + if err = p.notifier.Load(ctx, p.DB); err != nil { + req.Log.WithError(err).Error("unable to refresh notifier lists") + return from + } + for _, roomID := range newlyJoined { + roomUsers := p.notifier.JoinedUsers(roomID) + for i := range roomUsers { + // we already got a presence from this user + if _, ok := presences[roomUsers[i]]; ok { + continue + } + presences[roomUsers[i]], err = p.DB.GetPresence(ctx, roomUsers[i]) + if err != nil { + if err == sql.ErrNoRows { + continue + } + req.Log.WithError(err).Error("unable to query presence for user") + return from + } + } + } + } + + lastPos := to + for i := range presences { + presence := presences[i] + // Ignore users we don't share a room with + if req.Device.UserID != presence.UserID && !p.notifier.IsSharedUser(req.Device.UserID, presence.UserID) { + continue + } + cacheKey := req.Device.UserID + req.Device.ID + presence.UserID + pres, ok := p.cache.Load(cacheKey) + if ok { + // skip already sent presence + prevPresence := pres.(*types.PresenceInternal) + currentlyActive := prevPresence.CurrentlyActive() + skip := prevPresence.Equals(presence) && currentlyActive && req.Device.UserID != presence.UserID + if skip { + req.Log.Debugf("Skipping presence, no change (%s)", presence.UserID) + continue + } + } + + if _, known := types.PresenceFromString(presence.ClientFields.Presence); known { + presence.ClientFields.LastActiveAgo = presence.LastActiveAgo() + if presence.ClientFields.Presence == "online" { + currentlyActive := presence.CurrentlyActive() + presence.ClientFields.CurrentlyActive = ¤tlyActive + } + } else { + presence.ClientFields.Presence = "offline" + } + + content, err := json.Marshal(presence.ClientFields) + if err != nil { + return from + } + + req.Response.Presence.Events = append(req.Response.Presence.Events, gomatrixserverlib.ClientEvent{ + Content: content, + Sender: presence.UserID, + Type: gomatrixserverlib.MPresence, + }) + if presence.StreamPos > lastPos { + lastPos = presence.StreamPos + } + p.cache.Store(cacheKey, presence) + } + + return lastPos +} + +func joinedRooms(res *types.Response, userID string) []string { + var roomIDs []string + for roomID, join := range res.Rooms.Join { + // we would expect to see our join event somewhere if we newly joined the room. + // Normal events get put in the join section so it's not enough to know the room ID is present in 'join'. + newlyJoined := membershipEventPresent(join.State.Events, userID) + if newlyJoined { + roomIDs = append(roomIDs, roomID) + continue + } + newlyJoined = membershipEventPresent(join.Timeline.Events, userID) + if newlyJoined { + roomIDs = append(roomIDs, roomID) + } + } + return roomIDs +} + +func membershipEventPresent(events []gomatrixserverlib.ClientEvent, userID string) bool { + for _, ev := range events { + // it's enough to know that we have our member event here, don't need to check membership content + // as it's implied by being in the respective section of the sync response. + if ev.Type == gomatrixserverlib.MRoomMember && ev.StateKey != nil && *ev.StateKey == userID { + return true + } + } + return false +} diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go index 35ffd3a1e..9d7d479a2 100644 --- a/syncapi/streams/stream_receipt.go +++ b/syncapi/streams/stream_receipt.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" - eduAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -53,8 +52,12 @@ func (p *ReceiptStreamProvider) IncrementalSync( } // Group receipts by room, so we can create one ClientEvent for every room - receiptsByRoom := make(map[string][]eduAPI.OutputReceiptEvent) + receiptsByRoom := make(map[string][]types.OutputReceiptEvent) for _, receipt := range receipts { + // skip ignored user events + if _, ok := req.IgnoredUsers.List[receipt.UserID]; ok { + continue + } receiptsByRoom[receipt.RoomID] = append(receiptsByRoom[receipt.RoomID], receipt) } @@ -68,15 +71,15 @@ func (p *ReceiptStreamProvider) IncrementalSync( Type: gomatrixserverlib.MReceipt, RoomID: roomID, } - content := make(map[string]eduAPI.ReceiptMRead) + content := make(map[string]ReceiptMRead) for _, receipt := range receipts { read, ok := content[receipt.EventID] if !ok { - read = eduAPI.ReceiptMRead{ - User: make(map[string]eduAPI.ReceiptTS), + read = ReceiptMRead{ + User: make(map[string]ReceiptTS), } } - read.User[receipt.UserID] = eduAPI.ReceiptTS{TS: receipt.Timestamp} + read.User[receipt.UserID] = ReceiptTS{TS: receipt.Timestamp} content[receipt.EventID] = read } ev.Content, err = json.Marshal(content) @@ -91,3 +94,11 @@ func (p *ReceiptStreamProvider) IncrementalSync( return lastPos } + +type ReceiptMRead struct { + User map[string]ReceiptTS `json:"m.read"` +} + +type ReceiptTS struct { + TS gomatrixserverlib.Timestamp `json:"ts"` +} diff --git a/syncapi/streams/stream_sendtodevice.go b/syncapi/streams/stream_sendtodevice.go index a3aaf3d7d..6a18df506 100644 --- a/syncapi/streams/stream_sendtodevice.go +++ b/syncapi/streams/stream_sendtodevice.go @@ -48,6 +48,10 @@ func (p *SendToDeviceStreamProvider) IncrementalSync( // Add the updates into the sync response. for _, event := range events { + // skip ignored user events + if _, ok := req.IgnoredUsers.List[event.Sender]; ok { + continue + } req.Response.ToDevice.Events = append(req.Response.ToDevice.Events, event.SendToDeviceEvent) } } diff --git a/syncapi/streams/stream_typing.go b/syncapi/streams/stream_typing.go index 1e7a46bdc..f781065be 100644 --- a/syncapi/streams/stream_typing.go +++ b/syncapi/streams/stream_typing.go @@ -4,14 +4,14 @@ import ( "context" "encoding/json" - "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) type TypingStreamProvider struct { StreamProvider - EDUCache *cache.EDUCache + EDUCache *caching.EDUCache } func (p *TypingStreamProvider) CompleteSync( @@ -40,11 +40,18 @@ func (p *TypingStreamProvider) IncrementalSync( if users, updated := p.EDUCache.GetTypingUsersIfUpdatedAfter( roomID, int64(from), ); updated { + typingUsers := make([]string, 0, len(users)) + for i := range users { + // skip ignored user events + if _, ok := req.IgnoredUsers.List[users[i]]; !ok { + typingUsers = append(typingUsers, users[i]) + } + } ev := gomatrixserverlib.ClientEvent{ Type: gomatrixserverlib.MTyping, } ev.Content, err = json.Marshal(map[string]interface{}{ - "user_ids": users, + "user_ids": typingUsers, }) if err != nil { req.Log.WithError(err).Error("json.Marshal failed") diff --git a/syncapi/streams/streams.go b/syncapi/streams/streams.go index 17951acb4..c7d06a296 100644 --- a/syncapi/streams/streams.go +++ b/syncapi/streams/streams.go @@ -3,9 +3,10 @@ package streams import ( "context" - "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/internal/caching" keyapi "github.com/matrix-org/dendrite/keyserver/api" rsapi "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/syncapi/notifier" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -20,16 +21,18 @@ type Streams struct { AccountDataStreamProvider types.StreamProvider DeviceListStreamProvider types.StreamProvider NotificationDataStreamProvider types.StreamProvider + PresenceStreamProvider types.StreamProvider } func NewSyncStreamProviders( d storage.Database, userAPI userapi.UserInternalAPI, rsAPI rsapi.RoomserverInternalAPI, keyAPI keyapi.KeyInternalAPI, - eduCache *cache.EDUCache, + eduCache *caching.EDUCache, notifier *notifier.Notifier, ) *Streams { streams := &Streams{ PDUStreamProvider: &PDUStreamProvider{ StreamProvider: StreamProvider{DB: d}, + userAPI: userAPI, }, TypingStreamProvider: &TypingStreamProvider{ StreamProvider: StreamProvider{DB: d}, @@ -56,6 +59,10 @@ func NewSyncStreamProviders( rsAPI: rsAPI, keyAPI: keyAPI, }, + PresenceStreamProvider: &PresenceStreamProvider{ + StreamProvider: StreamProvider{DB: d}, + notifier: notifier, + }, } streams.PDUStreamProvider.Setup() @@ -66,6 +73,7 @@ func NewSyncStreamProviders( streams.AccountDataStreamProvider.Setup() streams.NotificationDataStreamProvider.Setup() streams.DeviceListStreamProvider.Setup() + streams.PresenceStreamProvider.Setup() return streams } @@ -80,5 +88,6 @@ func (s *Streams) Latest(ctx context.Context) types.StreamingToken { AccountDataPosition: s.AccountDataStreamProvider.LatestPosition(ctx), NotificationDataPosition: s.NotificationDataStreamProvider.LatestPosition(ctx), DeviceListPosition: s.DeviceListStreamProvider.LatestPosition(ctx), + PresencePosition: s.PresenceStreamProvider.LatestPosition(ctx), } } diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 2c9920d18..703340997 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -17,6 +17,8 @@ package sync import ( + "context" + "database/sql" "net" "net/http" "strings" @@ -33,8 +35,10 @@ import ( "github.com/matrix-org/dendrite/syncapi/streams" "github.com/matrix-org/dendrite/syncapi/types" userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" ) // RequestPool manages HTTP long-poll connections for /sync @@ -44,9 +48,15 @@ type RequestPool struct { userAPI userapi.UserInternalAPI keyAPI keyapi.KeyInternalAPI rsAPI roomserverAPI.RoomserverInternalAPI - lastseen sync.Map + lastseen *sync.Map + presence *sync.Map streams *streams.Streams Notifier *notifier.Notifier + producer PresencePublisher +} + +type PresencePublisher interface { + SendPresence(userID string, presence types.Presence, statusMsg *string) error } // NewRequestPool makes a new RequestPool @@ -55,18 +65,25 @@ func NewRequestPool( userAPI userapi.UserInternalAPI, keyAPI keyapi.KeyInternalAPI, rsAPI roomserverAPI.RoomserverInternalAPI, streams *streams.Streams, notifier *notifier.Notifier, + producer PresencePublisher, ) *RequestPool { + prometheus.MustRegister( + activeSyncRequests, waitingSyncRequests, + ) rp := &RequestPool{ db: db, cfg: cfg, userAPI: userAPI, keyAPI: keyAPI, rsAPI: rsAPI, - lastseen: sync.Map{}, + lastseen: &sync.Map{}, + presence: &sync.Map{}, streams: streams, Notifier: notifier, + producer: producer, } go rp.cleanLastSeen() + go rp.cleanPresence(db, time.Minute*5) return rp } @@ -80,6 +97,66 @@ func (rp *RequestPool) cleanLastSeen() { } } +func (rp *RequestPool) cleanPresence(db storage.Presence, cleanupTime time.Duration) { + if !rp.cfg.Matrix.Presence.EnableOutbound { + return + } + for { + rp.presence.Range(func(key interface{}, v interface{}) bool { + p := v.(types.PresenceInternal) + if time.Since(p.LastActiveTS.Time()) > cleanupTime { + rp.updatePresence(db, types.PresenceUnavailable.String(), p.UserID) + rp.presence.Delete(key) + } + return true + }) + time.Sleep(cleanupTime) + } +} + +// updatePresence sends presence updates to the SyncAPI and FederationAPI +func (rp *RequestPool) updatePresence(db storage.Presence, presence string, userID string) { + if !rp.cfg.Matrix.Presence.EnableOutbound { + return + } + if presence == "" { + presence = types.PresenceOnline.String() + } + + presenceID, ok := types.PresenceFromString(presence) + if !ok { // this should almost never happen + return + } + newPresence := types.PresenceInternal{ + ClientFields: types.PresenceClientResponse{ + Presence: presenceID.String(), + }, + Presence: presenceID, + UserID: userID, + LastActiveTS: gomatrixserverlib.AsTimestamp(time.Now()), + } + defer rp.presence.Store(userID, newPresence) + // avoid spamming presence updates when syncing + existingPresence, ok := rp.presence.LoadOrStore(userID, newPresence) + if ok { + p := existingPresence.(types.PresenceInternal) + if p.ClientFields.Presence == newPresence.ClientFields.Presence { + return + } + } + + // ensure we also send the current status_msg to federated servers and not nil + dbPresence, err := db.GetPresence(context.Background(), userID) + if err != nil && err != sql.ErrNoRows { + return + } + + if err := rp.producer.SendPresence(userID, presenceID, dbPresence.ClientFields.StatusMsg); err != nil { + logrus.WithError(err).Error("Unable to publish presence message from sync") + return + } +} + func (rp *RequestPool) updateLastSeen(req *http.Request, device *userapi.Device) { if _, ok := rp.lastseen.LoadOrStore(device.UserID+device.ID, struct{}{}); ok { return @@ -109,12 +186,6 @@ func (rp *RequestPool) updateLastSeen(req *http.Request, device *userapi.Device) rp.lastseen.Store(device.UserID+device.ID, time.Now()) } -func init() { - prometheus.MustRegister( - activeSyncRequests, waitingSyncRequests, - ) -} - var activeSyncRequests = prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: "dendrite", @@ -156,6 +227,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. defer activeSyncRequests.Dec() rp.updateLastSeen(req, device) + rp.updatePresence(rp.db, req.FormValue("set_presence"), device.UserID) waitingSyncRequests.Inc() defer waitingSyncRequests.Dec() @@ -219,6 +291,9 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. DeviceListPosition: rp.streams.DeviceListStreamProvider.CompleteSync( syncReq.Context, syncReq, ), + PresencePosition: rp.streams.PresenceStreamProvider.CompleteSync( + syncReq.Context, syncReq, + ), } } else { // Incremental sync @@ -255,6 +330,10 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. syncReq.Context, syncReq, syncReq.Since.DeviceListPosition, currentPos.DeviceListPosition, ), + PresencePosition: rp.streams.PresenceStreamProvider.IncrementalSync( + syncReq.Context, syncReq, + syncReq.Since.PresencePosition, currentPos.PresencePosition, + ), } } diff --git a/syncapi/sync/requestpool_test.go b/syncapi/sync/requestpool_test.go new file mode 100644 index 000000000..a80089945 --- /dev/null +++ b/syncapi/sync/requestpool_test.go @@ -0,0 +1,128 @@ +package sync + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/gomatrixserverlib" +) + +type dummyPublisher struct { + count int +} + +func (d *dummyPublisher) SendPresence(userID string, presence types.Presence, statusMsg *string) error { + d.count++ + return nil +} + +type dummyDB struct{} + +func (d dummyDB) UpdatePresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (types.StreamPosition, error) { + return 0, nil +} + +func (d dummyDB) GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error) { + return &types.PresenceInternal{}, nil +} + +func (d dummyDB) PresenceAfter(ctx context.Context, after types.StreamPosition) (map[string]*types.PresenceInternal, error) { + return map[string]*types.PresenceInternal{}, nil +} + +func (d dummyDB) MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error) { + return 0, nil +} + +func TestRequestPool_updatePresence(t *testing.T) { + type args struct { + presence string + userID string + sleep time.Duration + } + publisher := &dummyPublisher{} + syncMap := sync.Map{} + + tests := []struct { + name string + args args + wantIncrease bool + }{ + { + name: "new presence is published", + wantIncrease: true, + args: args{ + userID: "dummy", + }, + }, + { + name: "presence not published, no change", + args: args{ + userID: "dummy", + }, + }, + { + name: "new presence is published dummy2", + wantIncrease: true, + args: args{ + userID: "dummy2", + presence: "online", + }, + }, + { + name: "different presence is published dummy2", + wantIncrease: true, + args: args{ + userID: "dummy2", + presence: "unavailable", + }, + }, + { + name: "same presence is not published dummy2", + args: args{ + userID: "dummy2", + presence: "unavailable", + sleep: time.Millisecond * 150, + }, + }, + { + name: "same presence is published after being deleted", + wantIncrease: true, + args: args{ + userID: "dummy2", + presence: "unavailable", + }, + }, + } + rp := &RequestPool{ + presence: &syncMap, + producer: publisher, + cfg: &config.SyncAPI{ + Matrix: &config.Global{ + JetStream: config.JetStream{ + TopicPrefix: "Dendrite", + }, + Presence: config.PresenceOptions{ + EnableInbound: true, + EnableOutbound: true, + }, + }, + }, + } + db := dummyDB{} + go rp.cleanPresence(db, time.Millisecond*50) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + beforeCount := publisher.count + rp.updatePresence(db, tt.args.presence, tt.args.userID) + if tt.wantIncrease && publisher.count <= beforeCount { + t.Fatalf("expected count to increase: %d <= %d", publisher.count, beforeCount) + } + time.Sleep(tt.args.sleep) + }) + } +} diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index ed8118bfc..384121a8a 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -18,9 +18,9 @@ import ( "context" "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/internal/caching" "github.com/sirupsen/logrus" - "github.com/matrix-org/dendrite/eduserver/cache" keyapi "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" @@ -49,21 +49,27 @@ func AddPublicRoutes( federation *gomatrixserverlib.FederationClient, cfg *config.SyncAPI, ) { - js, _ := jetstream.Prepare(process, &cfg.Matrix.JetStream) + js, natsClient := jetstream.Prepare(process, &cfg.Matrix.JetStream) syncDB, err := storage.NewSyncServerDatasource(&cfg.Database) if err != nil { logrus.WithError(err).Panicf("failed to connect to sync db") } - eduCache := cache.New() - streams := streams.NewSyncStreamProviders(syncDB, userAPI, rsAPI, keyAPI, eduCache) - notifier := notifier.NewNotifier(streams.Latest(context.Background())) + eduCache := caching.NewTypingCache() + notifier := notifier.NewNotifier() + streams := streams.NewSyncStreamProviders(syncDB, userAPI, rsAPI, keyAPI, eduCache, notifier) + notifier.SetCurrentPosition(streams.Latest(context.Background())) if err = notifier.Load(context.Background(), syncDB); err != nil { logrus.WithError(err).Panicf("failed to load notifier ") } - requestPool := sync.NewRequestPool(syncDB, cfg, userAPI, keyAPI, rsAPI, streams, notifier) + federationPresenceProducer := &producers.FederationAPIPresenceProducer{ + Topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent), + JetStream: js, + } + + requestPool := sync.NewRequestPool(syncDB, cfg, userAPI, keyAPI, rsAPI, streams, notifier, federationPresenceProducer) userAPIStreamEventProducer := &producers.UserAPIStreamEventProducer{ JetStream: js, @@ -75,8 +81,6 @@ func AddPublicRoutes( Topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputReadUpdate), } - _ = userAPIReadUpdateProducer - keyChangeConsumer := consumers.NewOutputKeyChangeEventConsumer( process, cfg, cfg.Matrix.JetStream.Prefixed(jetstream.OutputKeyChangeEvent), js, keyAPI, rsAPI, syncDB, notifier, @@ -110,7 +114,7 @@ func AddPublicRoutes( } typingConsumer := consumers.NewOutputTypingEventConsumer( - process, cfg, js, syncDB, eduCache, notifier, streams.TypingStreamProvider, + process, cfg, js, eduCache, notifier, streams.TypingStreamProvider, ) if err = typingConsumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start typing consumer") @@ -131,5 +135,14 @@ func AddPublicRoutes( logrus.WithError(err).Panicf("failed to start receipts consumer") } + presenceConsumer := consumers.NewPresenceConsumer( + process, cfg, js, natsClient, syncDB, + notifier, streams.PresenceStreamProvider, + userAPI, + ) + if err = presenceConsumer.Start(); err != nil { + logrus.WithError(err).Panicf("failed to start presence consumer") + } + routing.Setup(router, requestPool, syncDB, userAPI, federation, rsAPI, cfg) } diff --git a/syncapi/types/presence.go b/syncapi/types/presence.go new file mode 100644 index 000000000..30e025b9f --- /dev/null +++ b/syncapi/types/presence.go @@ -0,0 +1,91 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "strings" + "time" + + "github.com/matrix-org/gomatrixserverlib" +) + +type Presence uint8 + +const ( + PresenceUnknown Presence = iota + PresenceUnavailable // unavailable + PresenceOnline // online + PresenceOffline // offline +) + +func (p Presence) String() string { + switch p { + case PresenceUnavailable: + return "unavailable" + case PresenceOnline: + return "online" + case PresenceOffline: + return "offline" + default: + return "unknown" + } +} + +// PresenceFromString returns the integer representation of the given input presence. +// Returns false for ok, if input is not a valid presence value. +func PresenceFromString(input string) (Presence, bool) { + switch strings.ToLower(input) { + case "unavailable": + return PresenceUnavailable, true + case "online": + return PresenceOnline, true + case "offline": + return PresenceOffline, true + default: + return PresenceUnknown, false + } +} + +type PresenceInternal struct { + ClientFields PresenceClientResponse + StreamPos StreamPosition `json:"-"` + UserID string `json:"-"` + LastActiveTS gomatrixserverlib.Timestamp `json:"-"` + Presence Presence `json:"-"` +} + +// Equals compares p1 with p2. +func (p1 *PresenceInternal) Equals(p2 *PresenceInternal) bool { + return p1.ClientFields.Presence == p2.ClientFields.Presence && + p1.ClientFields.StatusMsg == p2.ClientFields.StatusMsg && + p1.UserID == p2.UserID +} + +// CurrentlyActive returns the current active state. +func (p *PresenceInternal) CurrentlyActive() bool { + return time.Since(p.LastActiveTS.Time()).Minutes() < 5 +} + +// LastActiveAgo returns the time since the LastActiveTS in milliseconds. +func (p *PresenceInternal) LastActiveAgo() int64 { + return time.Since(p.LastActiveTS.Time()).Milliseconds() +} + +type PresenceClientResponse struct { + CurrentlyActive *bool `json:"currently_active,omitempty"` + LastActiveAgo int64 `json:"last_active_ago,omitempty"` + Presence string `json:"presence"` + StatusMsg *string `json:"status_msg,omitempty"` +} diff --git a/syncapi/types/provider.go b/syncapi/types/provider.go index f6185fcb5..e6777f643 100644 --- a/syncapi/types/provider.go +++ b/syncapi/types/provider.go @@ -21,6 +21,8 @@ type SyncRequest struct { // Updated by the PDU stream. Rooms map[string]string + // Updated by the PDU stream. + IgnoredUsers IgnoredUsers } type StreamProvider interface { diff --git a/syncapi/types/types.go b/syncapi/types/types.go index 4150e6c98..ba6b4f8cd 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -103,6 +103,7 @@ type StreamingToken struct { AccountDataPosition StreamPosition DeviceListPosition StreamPosition NotificationDataPosition StreamPosition + PresencePosition StreamPosition } // This will be used as a fallback by json.Marshal. @@ -118,11 +119,12 @@ func (s *StreamingToken) UnmarshalText(text []byte) (err error) { func (t StreamingToken) String() string { posStr := fmt.Sprintf( - "s%d_%d_%d_%d_%d_%d_%d_%d", + "s%d_%d_%d_%d_%d_%d_%d_%d_%d", t.PDUPosition, t.TypingPosition, t.ReceiptPosition, t.SendToDevicePosition, t.InvitePosition, t.AccountDataPosition, t.DeviceListPosition, t.NotificationDataPosition, + t.PresencePosition, ) return posStr } @@ -146,12 +148,14 @@ func (t *StreamingToken) IsAfter(other StreamingToken) bool { return true case t.NotificationDataPosition > other.NotificationDataPosition: return true + case t.PresencePosition > other.PresencePosition: + return true } return false } func (t *StreamingToken) IsEmpty() bool { - return t == nil || t.PDUPosition+t.TypingPosition+t.ReceiptPosition+t.SendToDevicePosition+t.InvitePosition+t.AccountDataPosition+t.DeviceListPosition+t.NotificationDataPosition == 0 + return t == nil || t.PDUPosition+t.TypingPosition+t.ReceiptPosition+t.SendToDevicePosition+t.InvitePosition+t.AccountDataPosition+t.DeviceListPosition+t.NotificationDataPosition+t.PresencePosition == 0 } // WithUpdates returns a copy of the StreamingToken with updates applied from another StreamingToken. @@ -192,6 +196,9 @@ func (t *StreamingToken) ApplyUpdates(other StreamingToken) { if other.NotificationDataPosition > t.NotificationDataPosition { t.NotificationDataPosition = other.NotificationDataPosition } + if other.PresencePosition > t.PresencePosition { + t.PresencePosition = other.PresencePosition + } } type TopologyToken struct { @@ -284,7 +291,7 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) { // s478_0_0_0_0_13.dl-0-2 but we have now removed partitioned stream positions tok = strings.Split(tok, ".")[0] parts := strings.Split(tok[1:], "_") - var positions [8]StreamPosition + var positions [9]StreamPosition for i, p := range parts { if i >= len(positions) { break @@ -306,6 +313,7 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) { AccountDataPosition: positions[5], DeviceListPosition: positions[6], NotificationDataPosition: positions[7], + PresencePosition: positions[8], } return token, nil } @@ -377,6 +385,11 @@ func (r *Response) IsEmpty() bool { // JoinResponse represents a /sync response for a room which is under the 'join' or 'peek' key. type JoinResponse struct { + Summary struct { + Heroes []string `json:"m.heroes,omitempty"` + JoinedMemberCount *int `json:"m.joined_member_count,omitempty"` + InvitedMemberCount *int `json:"m.invited_member_count,omitempty"` + } `json:"summary"` State struct { Events []gomatrixserverlib.ClientEvent `json:"events"` } `json:"state"` @@ -487,3 +500,25 @@ type StreamedEvent struct { Event *gomatrixserverlib.HeaderedEvent `json:"event"` StreamPosition StreamPosition `json:"stream_position"` } + +// OutputReceiptEvent is an entry in the receipt output kafka log +type OutputReceiptEvent struct { + UserID string `json:"user_id"` + RoomID string `json:"room_id"` + EventID string `json:"event_id"` + Type string `json:"type"` + Timestamp gomatrixserverlib.Timestamp `json:"timestamp"` +} + +// OutputSendToDeviceEvent is an entry in the send-to-device output kafka log. +// This contains the full event content, along with the user ID and device ID +// to which it is destined. +type OutputSendToDeviceEvent struct { + UserID string `json:"user_id"` + DeviceID string `json:"device_id"` + gomatrixserverlib.SendToDeviceEvent +} + +type IgnoredUsers struct { + List map[string]interface{} `json:"ignored_users"` +} diff --git a/syncapi/types/types_test.go b/syncapi/types/types_test.go index ff78bfb9d..19fcfc150 100644 --- a/syncapi/types/types_test.go +++ b/syncapi/types/types_test.go @@ -9,10 +9,10 @@ import ( func TestSyncTokens(t *testing.T) { shouldPass := map[string]string{ - "s4_0_0_0_0_0_0_0": StreamingToken{4, 0, 0, 0, 0, 0, 0, 0}.String(), - "s3_1_0_0_0_0_2_0": StreamingToken{3, 1, 0, 0, 0, 0, 2, 0}.String(), - "s3_1_2_3_5_0_0_0": StreamingToken{3, 1, 2, 3, 5, 0, 0, 0}.String(), - "t3_1": TopologyToken{3, 1}.String(), + "s4_0_0_0_0_0_0_0_3": StreamingToken{4, 0, 0, 0, 0, 0, 0, 0, 3}.String(), + "s3_1_0_0_0_0_2_0_5": StreamingToken{3, 1, 0, 0, 0, 0, 2, 0, 5}.String(), + "s3_1_2_3_5_0_0_0_6": StreamingToken{3, 1, 2, 3, 5, 0, 0, 0, 6}.String(), + "t3_1": TopologyToken{3, 1}.String(), } for a, b := range shouldPass { diff --git a/sytest-blacklist b/sytest-blacklist index 6a3b88390..f1bd60db1 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -1,35 +1,42 @@ # Blacklisted until matrix-org/dendrite#862 is reverted due to Riot bug + Latest account data appears in v2 /sync -# Blacklisted because we don't support ignores yet -Ignore invite in incremental sync - # Relies on a rejected PL event which will never be accepted into the DAG -# Caused by https://github.com/matrix-org/sytest/pull/911 + +# Caused by + Outbound federation requests missing prev_events and then asks for /state_ids and resolves the state -# We don't implement lazy membership loading yet. +# We don't implement lazy membership loading yet + The only membership state included in a gapped incremental sync is for senders in the timeline # Blacklisted out of flakiness after #1479 + Invited user can reject local invite after originator leaves Invited user can reject invite for empty room If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes # Blacklisted due to flakiness + Forgotten room messages cannot be paginated # Blacklisted due to flakiness after #1774 + Local device key changes get to remote servers with correct prev_id # Flakey + Local device key changes appear in /keys/changes # we don't support groups + Remove group category Remove group role # Flakey + AS-ghosted users can use rooms themselves AS-ghosted users can use rooms via AS Events in rooms with AS-hosted room aliases are sent to AS server @@ -37,6 +44,12 @@ Inviting an AS-hosted user asks the AS server Accesing an AS-hosted room alias asks the AS server # Flakey, need additional investigation + Messages that notify from another user increment notification_count Messages that highlight from another user increment unread highlight count Notifications can be viewed with GET /notifications + +# More flakey + +If remote user leaves room we no longer receive device updates +Local device key changes get to remote servers diff --git a/sytest-whitelist b/sytest-whitelist index 40bf5afac..dc67c9935 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -284,8 +284,6 @@ local user can join room with version 4 remote user can join room with version 3 remote user can join room with version 4 Remote user can backfill in a room with version 4 -# We don't support ignores yet, so ignore this for now - ha ha. -# Ignore invite in incremental sync Outbound federation can send invites via v2 API User can invite local user to room with version 3 User can invite local user to room with version 4 @@ -661,3 +659,41 @@ Canonical alias can include alt_aliases Can delete canonical alias AS can make room aliases /context/ with lazy_load_members filter works +/upgrade creates a new room +/upgrade should preserve room visibility for public rooms +/upgrade should preserve room visibility for private rooms +/upgrade copies the power levels to the new room +/upgrade preserves the power level of the upgrading user in old and new rooms +/upgrade copies important state to the new room +/upgrade copies ban events to the new room +local user has push rules copied to upgraded room +remote user has push rules copied to upgraded room +/upgrade moves aliases to the new room +/upgrade preserves room federation ability +/upgrade restricts power levels in the old room +/upgrade restricts power levels in the old room when the old PLs are unusual +/upgrade to an unknown version is rejected +/upgrade is rejected if the user can't send state events +/upgrade of a bogus room fails gracefully +Cannot send tombstone event that points to the same room +Room summary counts change when membership changes +GET /presence/:user_id/status fetches initial status +PUT /presence/:user_id/status updates my presence +Presence change reports an event to myself +Existing members see new members' presence +#Existing members see new member's presence +Newly joined room includes presence in incremental sync +Get presence for newly joined members in incremental sync +User sees their own presence in a sync +User sees updates to presence from other users in the incremental sync. +Presence changes are reported to local room members +Presence changes are also reported to remote room members +Presence changes to UNAVAILABLE are reported to local room members +Presence changes to UNAVAILABLE are reported to remote room members +New federated private chats get full presence information (SYN-115) +/upgrade copies >100 power levels to the new room +Room state after a rejected message event is the same as before +Room state after a rejected state event is the same as before +Ignore user in existing room +Ignore invite in full sync +Ignore invite in incremental sync \ No newline at end of file diff --git a/test/db.go b/test/db.go new file mode 100644 index 000000000..674fdf5c3 --- /dev/null +++ b/test/db.go @@ -0,0 +1,128 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "database/sql" + "fmt" + "os" + "os/exec" + "os/user" + "testing" +) + +type DBType int + +var DBTypeSQLite DBType = 1 +var DBTypePostgres DBType = 2 + +var Quiet = false + +func createLocalDB(dbName string) string { + if !Quiet { + fmt.Println("Note: tests require a postgres install accessible to the current user") + } + createDB := exec.Command("createdb", dbName) + if !Quiet { + createDB.Stdout = os.Stdout + createDB.Stderr = os.Stderr + } + err := createDB.Run() + if err != nil && !Quiet { + fmt.Println("createLocalDB returned error:", err) + } + return dbName +} + +func currentUser() string { + user, err := user.Current() + if err != nil { + if !Quiet { + fmt.Println("cannot get current user: ", err) + } + os.Exit(2) + } + return user.Username +} + +// Prepare a sqlite or postgres connection string for testing. +// Returns the connection string to use and a close function which must be called when the test finishes. +// Calling this function twice will return the same database, which will have data from previous tests +// unless close() is called. +// TODO: namespace for concurrent package tests +func PrepareDBConnectionString(t *testing.T, dbType DBType) (connStr string, close func()) { + if dbType == DBTypeSQLite { + dbname := "dendrite_test.db" + return fmt.Sprintf("file:%s", dbname), func() { + err := os.Remove(dbname) + if err != nil { + t.Fatalf("failed to cleanup sqlite db '%s': %s", dbname, err) + } + } + } + + // Required vars: user and db + // We'll try to infer from the local env if they are missing + user := os.Getenv("POSTGRES_USER") + if user == "" { + user = currentUser() + } + dbName := os.Getenv("POSTGRES_DB") + if dbName == "" { + dbName = createLocalDB("dendrite_test") + } + connStr = fmt.Sprintf( + "user=%s dbname=%s sslmode=disable", + user, dbName, + ) + // optional vars, used in CI + password := os.Getenv("POSTGRES_PASSWORD") + if password != "" { + connStr += fmt.Sprintf(" password=%s", password) + } + host := os.Getenv("POSTGRES_HOST") + if host != "" { + connStr += fmt.Sprintf(" host=%s", host) + } + + return connStr, func() { + // Drop all tables on the database to get a fresh instance + db, err := sql.Open("postgres", connStr) + if err != nil { + t.Fatalf("failed to connect to postgres db '%s': %s", connStr, err) + } + _, err = db.Exec(`DROP SCHEMA public CASCADE; + CREATE SCHEMA public;`) + if err != nil { + t.Fatalf("failed to cleanup postgres db '%s': %s", connStr, err) + } + _ = db.Close() + } +} + +// Creates subtests with each known DBType +func WithAllDatabases(t *testing.T, testFn func(t *testing.T, db DBType)) { + dbs := map[string]DBType{ + "postgres": DBTypePostgres, + "sqlite": DBTypeSQLite, + } + for dbName, dbType := range dbs { + dbt := dbType + t.Run(dbName, func(tt *testing.T) { + tt.Parallel() + testFn(tt, dbt) + }) + } +} diff --git a/test/event.go b/test/event.go new file mode 100644 index 000000000..b2e2805ba --- /dev/null +++ b/test/event.go @@ -0,0 +1,90 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "bytes" + "crypto/ed25519" + "testing" + "time" + + "github.com/matrix-org/gomatrixserverlib" +) + +type eventMods struct { + originServerTS time.Time + origin gomatrixserverlib.ServerName + stateKey *string + unsigned interface{} + keyID gomatrixserverlib.KeyID + privKey ed25519.PrivateKey +} + +type eventModifier func(e *eventMods) + +func WithTimestamp(ts time.Time) eventModifier { + return func(e *eventMods) { + e.originServerTS = ts + } +} + +func WithStateKey(skey string) eventModifier { + return func(e *eventMods) { + e.stateKey = &skey + } +} + +func WithUnsigned(unsigned interface{}) eventModifier { + return func(e *eventMods) { + e.unsigned = unsigned + } +} + +// Reverse a list of events +func Reversed(in []*gomatrixserverlib.HeaderedEvent) []*gomatrixserverlib.HeaderedEvent { + out := make([]*gomatrixserverlib.HeaderedEvent, len(in)) + for i := 0; i < len(in); i++ { + out[i] = in[len(in)-i-1] + } + return out +} + +func AssertEventIDsEqual(t *testing.T, gotEventIDs []string, wants []*gomatrixserverlib.HeaderedEvent) { + t.Helper() + if len(gotEventIDs) != len(wants) { + t.Fatalf("length mismatch: got %d events, want %d", len(gotEventIDs), len(wants)) + } + for i := range wants { + w := wants[i].EventID() + g := gotEventIDs[i] + if w != g { + t.Errorf("event at index %d mismatch:\ngot %s\n\nwant %s", i, string(g), string(w)) + } + } +} + +func AssertEventsEqual(t *testing.T, gots, wants []*gomatrixserverlib.HeaderedEvent) { + t.Helper() + if len(gots) != len(wants) { + t.Fatalf("length mismatch: got %d events, want %d", len(gots), len(wants)) + } + for i := range wants { + w := wants[i].JSON() + g := gots[i].JSON() + if !bytes.Equal(w, g) { + t.Errorf("event at index %d mismatch:\ngot %s\n\nwant %s", i, string(g), string(w)) + } + } +} diff --git a/test/room.go b/test/room.go new file mode 100644 index 000000000..619cb5c9a --- /dev/null +++ b/test/room.go @@ -0,0 +1,223 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "crypto/ed25519" + "encoding/json" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/matrix-org/dendrite/internal/eventutil" + "github.com/matrix-org/gomatrixserverlib" +) + +type Preset int + +var ( + PresetNone Preset = 0 + PresetPrivateChat Preset = 1 + PresetPublicChat Preset = 2 + PresetTrustedPrivateChat Preset = 3 + + roomIDCounter = int64(0) + + testKeyID = gomatrixserverlib.KeyID("ed25519:test") + testPrivateKey = ed25519.NewKeyFromSeed([]byte{ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + }) +) + +type Room struct { + ID string + Version gomatrixserverlib.RoomVersion + preset Preset + creator *User + + authEvents gomatrixserverlib.AuthEvents + events []*gomatrixserverlib.HeaderedEvent +} + +// Create a new test room. Automatically creates the initial create events. +func NewRoom(t *testing.T, creator *User, modifiers ...roomModifier) *Room { + t.Helper() + counter := atomic.AddInt64(&roomIDCounter, 1) + + // set defaults then let roomModifiers override + r := &Room{ + ID: fmt.Sprintf("!%d:localhost", counter), + creator: creator, + authEvents: gomatrixserverlib.NewAuthEvents(nil), + preset: PresetPublicChat, + Version: gomatrixserverlib.RoomVersionV9, + } + for _, m := range modifiers { + m(t, r) + } + r.insertCreateEvents(t) + return r +} + +func (r *Room) insertCreateEvents(t *testing.T) { + t.Helper() + var joinRule gomatrixserverlib.JoinRuleContent + var hisVis gomatrixserverlib.HistoryVisibilityContent + plContent := eventutil.InitialPowerLevelsContent(r.creator.ID) + switch r.preset { + case PresetTrustedPrivateChat: + fallthrough + case PresetPrivateChat: + joinRule.JoinRule = "invite" + hisVis.HistoryVisibility = "shared" + case PresetPublicChat: + joinRule.JoinRule = "public" + hisVis.HistoryVisibility = "shared" + } + r.CreateAndInsert(t, r.creator, gomatrixserverlib.MRoomCreate, map[string]interface{}{ + "creator": r.creator.ID, + "room_version": r.Version, + }, WithStateKey("")) + r.CreateAndInsert(t, r.creator, gomatrixserverlib.MRoomMember, map[string]interface{}{ + "membership": "join", + }, WithStateKey(r.creator.ID)) + r.CreateAndInsert(t, r.creator, gomatrixserverlib.MRoomPowerLevels, plContent, WithStateKey("")) + r.CreateAndInsert(t, r.creator, gomatrixserverlib.MRoomJoinRules, joinRule, WithStateKey("")) + r.CreateAndInsert(t, r.creator, gomatrixserverlib.MRoomHistoryVisibility, hisVis, WithStateKey("")) +} + +// Create an event in this room but do not insert it. Does not modify the room in any way (depth, fwd extremities, etc) so is thread-safe. +func (r *Room) CreateEvent(t *testing.T, creator *User, eventType string, content interface{}, mods ...eventModifier) *gomatrixserverlib.HeaderedEvent { + t.Helper() + depth := 1 + len(r.events) // depth starts at 1 + + // possible event modifiers (optional fields) + mod := &eventMods{} + for _, m := range mods { + m(mod) + } + + if mod.privKey == nil { + mod.privKey = testPrivateKey + } + if mod.keyID == "" { + mod.keyID = testKeyID + } + if mod.originServerTS.IsZero() { + mod.originServerTS = time.Now() + } + if mod.origin == "" { + mod.origin = gomatrixserverlib.ServerName("localhost") + } + + var unsigned gomatrixserverlib.RawJSON + var err error + if mod.unsigned != nil { + unsigned, err = json.Marshal(mod.unsigned) + if err != nil { + t.Fatalf("CreateEvent[%s]: failed to marshal unsigned field: %s", eventType, err) + } + } + + builder := &gomatrixserverlib.EventBuilder{ + Sender: creator.ID, + RoomID: r.ID, + Type: eventType, + StateKey: mod.stateKey, + Depth: int64(depth), + Unsigned: unsigned, + } + err = builder.SetContent(content) + if err != nil { + t.Fatalf("CreateEvent[%s]: failed to SetContent: %s", eventType, err) + } + if depth > 1 { + builder.PrevEvents = []gomatrixserverlib.EventReference{r.events[len(r.events)-1].EventReference()} + } + + eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder) + if err != nil { + t.Fatalf("CreateEvent[%s]: failed to StateNeededForEventBuilder: %s", eventType, err) + } + refs, err := eventsNeeded.AuthEventReferences(&r.authEvents) + if err != nil { + t.Fatalf("CreateEvent[%s]: failed to AuthEventReferences: %s", eventType, err) + } + builder.AuthEvents = refs + ev, err := builder.Build( + mod.originServerTS, mod.origin, mod.keyID, + mod.privKey, r.Version, + ) + if err != nil { + t.Fatalf("CreateEvent[%s]: failed to build event: %s", eventType, err) + } + if err = gomatrixserverlib.Allowed(ev, &r.authEvents); err != nil { + t.Fatalf("CreateEvent[%s]: failed to verify event was allowed: %s", eventType, err) + } + return ev.Headered(r.Version) +} + +// Add a new event to this room DAG. Not thread-safe. +func (r *Room) InsertEvent(t *testing.T, he *gomatrixserverlib.HeaderedEvent) { + t.Helper() + // Add the event to the list of auth events + r.events = append(r.events, he) + if he.StateKey() != nil { + err := r.authEvents.AddEvent(he.Unwrap()) + if err != nil { + t.Fatalf("InsertEvent: failed to add event to auth events: %s", err) + } + } +} + +func (r *Room) Events() []*gomatrixserverlib.HeaderedEvent { + return r.events +} + +func (r *Room) CreateAndInsert(t *testing.T, creator *User, eventType string, content interface{}, mods ...eventModifier) *gomatrixserverlib.HeaderedEvent { + t.Helper() + he := r.CreateEvent(t, creator, eventType, content, mods...) + r.InsertEvent(t, he) + return he +} + +// All room modifiers are below + +type roomModifier func(t *testing.T, r *Room) + +func RoomPreset(p Preset) roomModifier { + return func(t *testing.T, r *Room) { + switch p { + case PresetPrivateChat: + fallthrough + case PresetPublicChat: + fallthrough + case PresetTrustedPrivateChat: + fallthrough + case PresetNone: + r.preset = p + default: + t.Errorf("invalid RoomPreset: %v", p) + } + } +} + +func RoomVersion(ver gomatrixserverlib.RoomVersion) roomModifier { + return func(t *testing.T, r *Room) { + r.Version = ver + } +} diff --git a/test/user.go b/test/user.go new file mode 100644 index 000000000..41a66e1c4 --- /dev/null +++ b/test/user.go @@ -0,0 +1,36 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "fmt" + "sync/atomic" +) + +var ( + userIDCounter = int64(0) +) + +type User struct { + ID string +} + +func NewUser() *User { + counter := atomic.AddInt64(&userIDCounter, 1) + u := &User{ + ID: fmt.Sprintf("@%d:localhost", counter), + } + return u +} diff --git a/userapi/api/api.go b/userapi/api/api.go index 513a060c4..b86774d14 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -31,12 +31,10 @@ type UserInternalAPI interface { UserRegisterAPI UserAccountAPI UserThreePIDAPI + UserDeviceAPI InputAccountData(ctx context.Context, req *InputAccountDataRequest, res *InputAccountDataResponse) error - PerformDeviceDeletion(ctx context.Context, req *PerformDeviceDeletionRequest, res *PerformDeviceDeletionResponse) error - PerformLastSeenUpdate(ctx context.Context, req *PerformLastSeenUpdateRequest, res *PerformLastSeenUpdateResponse) error - PerformDeviceUpdate(ctx context.Context, req *PerformDeviceUpdateRequest, res *PerformDeviceUpdateResponse) error PerformOpenIDTokenCreation(ctx context.Context, req *PerformOpenIDTokenCreationRequest, res *PerformOpenIDTokenCreationResponse) error PerformKeyBackup(ctx context.Context, req *PerformKeyBackupRequest, res *PerformKeyBackupResponse) error PerformPusherSet(ctx context.Context, req *PerformPusherSetRequest, res *struct{}) error @@ -45,15 +43,25 @@ type UserInternalAPI interface { QueryKeyBackup(ctx context.Context, req *QueryKeyBackupRequest, res *QueryKeyBackupResponse) QueryAccessToken(ctx context.Context, req *QueryAccessTokenRequest, res *QueryAccessTokenResponse) error - QueryDevices(ctx context.Context, req *QueryDevicesRequest, res *QueryDevicesResponse) error QueryAccountData(ctx context.Context, req *QueryAccountDataRequest, res *QueryAccountDataResponse) error - QueryDeviceInfos(ctx context.Context, req *QueryDeviceInfosRequest, res *QueryDeviceInfosResponse) error QueryOpenIDToken(ctx context.Context, req *QueryOpenIDTokenRequest, res *QueryOpenIDTokenResponse) error QueryPushers(ctx context.Context, req *QueryPushersRequest, res *QueryPushersResponse) error QueryPushRules(ctx context.Context, req *QueryPushRulesRequest, res *QueryPushRulesResponse) error QueryNotifications(ctx context.Context, req *QueryNotificationsRequest, res *QueryNotificationsResponse) error } +type UserDeviceAPI interface { + PerformDeviceDeletion(ctx context.Context, req *PerformDeviceDeletionRequest, res *PerformDeviceDeletionResponse) error + PerformLastSeenUpdate(ctx context.Context, req *PerformLastSeenUpdateRequest, res *PerformLastSeenUpdateResponse) error + PerformDeviceUpdate(ctx context.Context, req *PerformDeviceUpdateRequest, res *PerformDeviceUpdateResponse) error + QueryDevices(ctx context.Context, req *QueryDevicesRequest, res *QueryDevicesResponse) error + QueryDeviceInfos(ctx context.Context, req *QueryDeviceInfosRequest, res *QueryDeviceInfosResponse) error +} + +type UserDirectoryProvider interface { + QuerySearchProfiles(ctx context.Context, req *QuerySearchProfilesRequest, res *QuerySearchProfilesResponse) error +} + // UserProfileAPI provides functions for getting user profiles type UserProfileAPI interface { QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error diff --git a/userapi/consumers/syncapi_streamevent.go b/userapi/consumers/syncapi_streamevent.go index da3cd3937..9ef7b5083 100644 --- a/userapi/consumers/syncapi_streamevent.go +++ b/userapi/consumers/syncapi_streamevent.go @@ -184,6 +184,7 @@ func (s *OutputStreamEventConsumer) localRoomMembers(ctx context.Context, roomID req := &rsapi.QueryMembershipsForRoomRequest{ RoomID: roomID, JoinedOnly: true, + LocalOnly: true, } var res rsapi.QueryMembershipsForRoomResponse @@ -403,8 +404,24 @@ func (s *OutputStreamEventConsumer) evaluatePushRules(ctx context.Context, event return nil, nil } + // Get accountdata to check if the event.Sender() is ignored by mem.LocalPart + data, err := s.db.GetAccountDataByType(ctx, mem.Localpart, "", "m.ignored_user_list") + if err != nil { + return nil, err + } + if data != nil { + ignored := types.IgnoredUsers{} + err = json.Unmarshal(data, &ignored) + if err != nil { + return nil, err + } + sender := event.Sender() + if _, ok := ignored.List[sender]; ok { + return nil, fmt.Errorf("user %s is ignored", sender) + } + } var res api.QueryPushRulesResponse - if err := s.userAPI.QueryPushRules(ctx, &api.QueryPushRulesRequest{UserID: mem.UserID}, &res); err != nil { + if err = s.userAPI.QueryPushRules(ctx, &api.QueryPushRulesRequest{UserID: mem.UserID}, &res); err != nil { return nil, err } diff --git a/userapi/internal/api.go b/userapi/internal/api.go index afe57da25..206c6f7de 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -776,12 +776,8 @@ func (a *UserInternalAPI) QueryNumericLocalpart(ctx context.Context, res *api.Qu } func (a *UserInternalAPI) QueryAccountAvailability(ctx context.Context, req *api.QueryAccountAvailabilityRequest, res *api.QueryAccountAvailabilityResponse) error { - _, err := a.DB.CheckAccountAvailability(ctx, req.Localpart) - if err == sql.ErrNoRows { - res.Available = true - return nil - } - res.Available = false + var err error + res.Available, err = a.DB.CheckAccountAvailability(ctx, req.Localpart) return err } diff --git a/userapi/inthttp/server.go b/userapi/inthttp/server.go index f229aa3bb..ad532b901 100644 --- a/userapi/inthttp/server.go +++ b/userapi/inthttp/server.go @@ -369,6 +369,19 @@ func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + internalAPIMux.Handle(QueryAccountAvailabilityPath, + httputil.MakeInternalAPI("queryAccountAvailability", func(req *http.Request) util.JSONResponse { + request := api.QueryAccountAvailabilityRequest{} + response := api.QueryAccountAvailabilityResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := s.QueryAccountAvailability(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) internalAPIMux.Handle(QueryAccountByPasswordPath, httputil.MakeInternalAPI("queryAccountByPassword", func(req *http.Request) util.JSONResponse { request := api.QueryAccountByPasswordRequest{} diff --git a/userapi/storage/postgres/profile_table.go b/userapi/storage/postgres/profile_table.go index 32a4b5506..6d336eb8e 100644 --- a/userapi/storage/postgres/profile_table.go +++ b/userapi/storage/postgres/profile_table.go @@ -53,6 +53,7 @@ const selectProfilesBySearchSQL = "" + "SELECT localpart, display_name, avatar_url FROM account_profiles WHERE localpart LIKE $1 OR display_name LIKE $1 LIMIT $2" type profilesStatements struct { + serverNoticesLocalpart string insertProfileStmt *sql.Stmt selectProfileByLocalpartStmt *sql.Stmt setAvatarURLStmt *sql.Stmt @@ -60,8 +61,10 @@ type profilesStatements struct { selectProfilesBySearchStmt *sql.Stmt } -func NewPostgresProfilesTable(db *sql.DB) (tables.ProfileTable, error) { - s := &profilesStatements{} +func NewPostgresProfilesTable(db *sql.DB, serverNoticesLocalpart string) (tables.ProfileTable, error) { + s := &profilesStatements{ + serverNoticesLocalpart: serverNoticesLocalpart, + } _, err := db.Exec(profilesSchema) if err != nil { return nil, err @@ -126,7 +129,9 @@ func (s *profilesStatements) SelectProfilesBySearch( if err := rows.Scan(&profile.Localpart, &profile.DisplayName, &profile.AvatarURL); err != nil { return nil, err } - profiles = append(profiles, profile) + if profile.Localpart != s.serverNoticesLocalpart { + profiles = append(profiles, profile) + } } return profiles, nil } diff --git a/userapi/storage/postgres/storage.go b/userapi/storage/postgres/storage.go index c74a999f4..b2a517605 100644 --- a/userapi/storage/postgres/storage.go +++ b/userapi/storage/postgres/storage.go @@ -30,7 +30,7 @@ import ( ) // NewDatabase creates a new accounts and profiles database -func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration) (*shared.Database, error) { +func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (*shared.Database, error) { db, err := sqlutil.Open(dbProperties) if err != nil { return nil, err @@ -77,7 +77,7 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver if err != nil { return nil, fmt.Errorf("NewPostgresOpenIDTable: %w", err) } - profilesTable, err := NewPostgresProfilesTable(db) + profilesTable, err := NewPostgresProfilesTable(db, serverNoticesLocalpart) if err != nil { return nil, fmt.Errorf("NewPostgresProfilesTable: %w", err) } diff --git a/userapi/storage/sqlite3/profile_table.go b/userapi/storage/sqlite3/profile_table.go index d85b19c7b..3050ff4b5 100644 --- a/userapi/storage/sqlite3/profile_table.go +++ b/userapi/storage/sqlite3/profile_table.go @@ -54,6 +54,7 @@ const selectProfilesBySearchSQL = "" + type profilesStatements struct { db *sql.DB + serverNoticesLocalpart string insertProfileStmt *sql.Stmt selectProfileByLocalpartStmt *sql.Stmt setAvatarURLStmt *sql.Stmt @@ -61,9 +62,10 @@ type profilesStatements struct { selectProfilesBySearchStmt *sql.Stmt } -func NewSQLiteProfilesTable(db *sql.DB) (tables.ProfileTable, error) { +func NewSQLiteProfilesTable(db *sql.DB, serverNoticesLocalpart string) (tables.ProfileTable, error) { s := &profilesStatements{ - db: db, + db: db, + serverNoticesLocalpart: serverNoticesLocalpart, } _, err := db.Exec(profilesSchema) if err != nil { @@ -131,7 +133,9 @@ func (s *profilesStatements) SelectProfilesBySearch( if err := rows.Scan(&profile.Localpart, &profile.DisplayName, &profile.AvatarURL); err != nil { return nil, err } - profiles = append(profiles, profile) + if profile.Localpart != s.serverNoticesLocalpart { + profiles = append(profiles, profile) + } } return profiles, nil } diff --git a/userapi/storage/sqlite3/storage.go b/userapi/storage/sqlite3/storage.go index b5bb96c42..03c013f00 100644 --- a/userapi/storage/sqlite3/storage.go +++ b/userapi/storage/sqlite3/storage.go @@ -31,7 +31,7 @@ import ( ) // NewDatabase creates a new accounts and profiles database -func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration) (*shared.Database, error) { +func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (*shared.Database, error) { db, err := sqlutil.Open(dbProperties) if err != nil { return nil, err @@ -78,7 +78,7 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver if err != nil { return nil, fmt.Errorf("NewSQLiteOpenIDTable: %w", err) } - profilesTable, err := NewSQLiteProfilesTable(db) + profilesTable, err := NewSQLiteProfilesTable(db, serverNoticesLocalpart) if err != nil { return nil, fmt.Errorf("NewSQLiteProfilesTable: %w", err) } diff --git a/userapi/storage/storage.go b/userapi/storage/storage.go index 4711439af..f372fe7dc 100644 --- a/userapi/storage/storage.go +++ b/userapi/storage/storage.go @@ -30,12 +30,12 @@ import ( // NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) // and sets postgres connection parameters -func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration) (Database, error) { +func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (Database, error) { switch { case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.NewDatabase(dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime) + return sqlite3.NewDatabase(dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart) case dbProperties.ConnectionString.IsPostgres(): - return postgres.NewDatabase(dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime) + return postgres.NewDatabase(dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart) default: return nil, fmt.Errorf("unexpected database type") } diff --git a/userapi/storage/storage_wasm.go b/userapi/storage/storage_wasm.go index 701dcd833..779f77568 100644 --- a/userapi/storage/storage_wasm.go +++ b/userapi/storage/storage_wasm.go @@ -29,10 +29,11 @@ func NewDatabase( bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, + serverNoticesLocalpart string, ) (Database, error) { switch { case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.NewDatabase(dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime) + return sqlite3.NewDatabase(dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart) case dbProperties.ConnectionString.IsPostgres(): return nil, fmt.Errorf("can't use Postgres implementation") default: diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go index 25319c4bf..8c3608bd8 100644 --- a/userapi/userapi_test.go +++ b/userapi/userapi_test.go @@ -52,7 +52,7 @@ func MustMakeInternalAPI(t *testing.T, opts apiTestOpts) (api.UserInternalAPI, s MaxOpenConnections: 1, MaxIdleConnections: 1, } - accountDB, err := storage.NewDatabase(dbopts, serverName, bcrypt.MinCost, config.DefaultOpenIDTokenLifetimeMS, opts.loginTokenLifetime) + accountDB, err := storage.NewDatabase(dbopts, serverName, bcrypt.MinCost, config.DefaultOpenIDTokenLifetimeMS, opts.loginTokenLifetime, "") if err != nil { t.Fatalf("failed to create account DB: %s", err) }