From f76f28e6db85df145d54f4dab86be721ef641a83 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Thu, 7 Jul 2022 16:29:25 +0200 Subject: [PATCH 01/19] Fix issue `uint64 values with high bit are not supported` in presence (#2562) * Fix issue #2528 * Use gomatrixserverlib.Timestamp * Use ParseUint instead of ParseInt --- syncapi/consumers/presence.go | 8 ++++---- syncapi/sync/requestpool.go | 4 ++-- syncapi/sync/requestpool_test.go | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/syncapi/consumers/presence.go b/syncapi/consumers/presence.go index 0217e1956..db7d67fa6 100644 --- a/syncapi/consumers/presence.go +++ b/syncapi/consumers/presence.go @@ -144,7 +144,7 @@ func (s *PresenceConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool { return true } - ts, err := strconv.Atoi(timestamp) + ts, err := strconv.ParseUint(timestamp, 10, 64) if err != nil { return true } @@ -157,12 +157,12 @@ func (s *PresenceConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool { // already checked, so no need to check error p, _ := types.PresenceFromString(presence) - s.EmitPresence(ctx, userID, p, statusMsg, ts, fromSync) + s.EmitPresence(ctx, userID, p, statusMsg, gomatrixserverlib.Timestamp(ts), fromSync) return true } -func (s *PresenceConsumer) EmitPresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, ts int, fromSync bool) { - pos, err := s.db.UpdatePresence(ctx, userID, presence, statusMsg, gomatrixserverlib.Timestamp(ts), fromSync) +func (s *PresenceConsumer) EmitPresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, ts gomatrixserverlib.Timestamp, fromSync bool) { + pos, err := s.db.UpdatePresence(ctx, userID, presence, statusMsg, ts, fromSync) if err != nil { logrus.WithError(err).WithField("user", userID).WithField("presence", presence).Warn("failed to updated presence for user") return diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 7b9526b53..6f0849e08 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -61,7 +61,7 @@ type PresencePublisher interface { } type PresenceConsumer interface { - EmitPresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, ts int, fromSync bool) + EmitPresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, ts gomatrixserverlib.Timestamp, fromSync bool) } // NewRequestPool makes a new RequestPool @@ -171,7 +171,7 @@ func (rp *RequestPool) updatePresence(db storage.Presence, presence string, user // the /sync response else we may not return presence: online immediately. rp.consumer.EmitPresence( context.Background(), userID, presenceID, newPresence.ClientFields.StatusMsg, - int(gomatrixserverlib.AsTimestamp(time.Now())), true, + gomatrixserverlib.AsTimestamp(time.Now()), true, ) } diff --git a/syncapi/sync/requestpool_test.go b/syncapi/sync/requestpool_test.go index 0c7209521..48e6c6c7a 100644 --- a/syncapi/sync/requestpool_test.go +++ b/syncapi/sync/requestpool_test.go @@ -40,7 +40,7 @@ func (d dummyDB) MaxStreamPositionForPresence(ctx context.Context) (types.Stream type dummyConsumer struct{} -func (d dummyConsumer) EmitPresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, ts int, fromSync bool) { +func (d dummyConsumer) EmitPresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, ts gomatrixserverlib.Timestamp, fromSync bool) { } From eb8dc50a970cc1bfd82dc4bace76aba00181df6e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 8 Jul 2022 14:54:08 +0100 Subject: [PATCH 02/19] Update Pinecone to matrix-org/pinecone@1ce778f --- build/gobind-pinecone/monolith.go | 2 +- cmd/dendrite-demo-pinecone/main.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index b44e110ad..f3895ae23 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -239,7 +239,7 @@ func (m *DendriteMonolith) Start() { m.PineconeRouter = pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk, false) m.PineconeQUIC = pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), m.PineconeRouter, []string{"matrix"}) m.PineconeMulticast = pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), m.PineconeRouter) - m.PineconeManager = pineconeConnections.NewConnectionManager(m.PineconeRouter) + m.PineconeManager = pineconeConnections.NewConnectionManager(m.PineconeRouter, nil) prefix := hex.EncodeToString(pk) cfg := &config.Dendrite{} diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index 703436051..8fa935ddf 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -92,7 +92,7 @@ func main() { pRouter := pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk, false) pQUIC := pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), pRouter, []string{"matrix"}) pMulticast := pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), pRouter) - pManager := pineconeConnections.NewConnectionManager(pRouter) + pManager := pineconeConnections.NewConnectionManager(pRouter, nil) pMulticast.Start() if instancePeer != nil && *instancePeer != "" { pManager.AddPeer(*instancePeer) diff --git a/go.mod b/go.mod index 4c432611e..cf178cfcc 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 github.com/matrix-org/gomatrixserverlib v0.0.0-20220701090733-da53994b0c7f - github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 + github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.13 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect diff --git a/go.sum b/go.sum index 777a31228..3ff09b2dc 100644 --- a/go.sum +++ b/go.sum @@ -420,8 +420,8 @@ github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= github.com/matrix-org/gomatrixserverlib v0.0.0-20220701090733-da53994b0c7f h1:XF2+J6sOq07yhK1I7ItwsgRwXorjj7gqiCvgZ4dn8W8= github.com/matrix-org/gomatrixserverlib v0.0.0-20220701090733-da53994b0c7f/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk= -github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE= -github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc= +github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a h1:DdG8vXMlZ65EAtc4V+3t7zHZ2Gqs24pSnyXS+4BRHUs= +github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From 3ea21273bcc151b36eec412d0ec550642fe9b04f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 11 Jul 2022 14:31:31 +0100 Subject: [PATCH 03/19] Ristretto cache (#2563) * Try Ristretto cache * Tweak * It's beautiful * Update GMSL * More strict keyable interface * Fix that some more * Make less panicky * Don't enforce mutability checks for now * Determine mutability using deep equality * Tweaks * Namespace keys * Make federation caches mutable * Update cost estimation, add metric * Update GMSL * Estimate cost for metrics better * Reduce counters a bit * Try caching events * Some guards * Try again * Try this * Use separate caches for hopefully better hash distribution * Fix bug with admitting events into cache * Try to fix bugs * Check nil * Try that again * Preserve order jeezo this is messy * thanks VS Code for doing exactly the wrong thing * Try this again * Be more specific * aaaaargh * One more time * That might be better * Stronger sorting * Cache expiries, async publishing of EDUs * Put it back * Use a shared cache again * Cost estimation fixes * Update ristretto * Reduce counters a bit * Clean up a bit * Update GMSL * 1GB * Configurable cache sizees * Tweaks * Add `config.DataUnit` for specifying friendly cache sizes * Various tweaks * Update GMSL * Add back some lazy loading caching * Include key in cost * Include key in cost * Tweak max age handling, config key name * Only register prometheus metrics if requested * Review comments @S7evinK * Don't return errors when creating caches (it is better just to crash since otherwise we'll `nil`-pointer exception everywhere) * Review comments * Update sample configs * Update GHA Workflow * Update Complement images to Go 1.18 * Remove the cache test from the federation API as we no longer guarantee immediate cache admission * Don't check the caches in the renewal test * Possibly fix the upgrade tests * Update to matrix-org/gomatrixserverlib#322 * Update documentation to refer to Go 1.18 --- .github/workflows/dendrite.yml | 10 +- README.md | 2 +- build/scripts/Complement.Dockerfile | 8 +- build/scripts/ComplementLocal.Dockerfile | 28 +-- build/scripts/ComplementPostgres.Dockerfile | 34 +-- cmd/dendrite-demo-yggdrasil/README.md | 6 +- cmd/dendrite-upgrade-tests/main.go | 2 +- cmd/resolve-state/main.go | 11 +- dendrite-sample.monolith.yaml | 19 ++ dendrite-sample.polylith.yaml | 19 ++ docs/installation/1_planning.md | 2 +- federationapi/federationapi_keys_test.go | 118 +---------- federationapi/routing/invite.go | 12 +- federationapi/routing/join.go | 11 +- federationapi/routing/leave.go | 11 +- go.mod | 85 +++++--- go.sum | 174 +-------------- internal/caching/cache_federationevents.go | 39 +--- internal/caching/cache_lazy_load_members.go | 68 ++---- internal/caching/cache_roomevents.go | 21 ++ internal/caching/cache_roominfo.go | 17 +- internal/caching/cache_roomservernids.go | 20 +- internal/caching/cache_roomversions.go | 15 +- internal/caching/cache_serverkeys.go | 24 +-- internal/caching/cache_space_rooms.go | 17 +- internal/caching/caches.go | 52 +++-- internal/caching/impl_inmemorylru.go | 189 ----------------- internal/caching/impl_ristretto.go | 200 ++++++++++++++++++ internal/eventutil/events.go | 18 +- roomserver/internal/input/input_events.go | 4 +- roomserver/internal/input/input_test.go | 6 +- .../internal/perform/perform_backfill.go | 5 +- roomserver/state/state.go | 2 +- roomserver/storage/shared/storage.go | 39 +++- setup/base/base.go | 7 +- setup/config/config_global.go | 46 ++++ setup/config/config_test.go | 21 ++ syncapi/storage/shared/syncserver.go | 5 +- 38 files changed, 603 insertions(+), 764 deletions(-) create mode 100644 internal/caching/cache_roomevents.go delete mode 100644 internal/caching/impl_inmemorylru.go create mode 100644 internal/caching/impl_ristretto.go diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index 6ebef4e13..9b452d849 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -24,7 +24,7 @@ jobs: - name: Install Go uses: actions/setup-go@v2 with: - go-version: 1.16 + go-version: 1.18 - uses: actions/cache@v2 with: @@ -97,7 +97,7 @@ jobs: strategy: fail-fast: false matrix: - go: ["1.16", "1.17", "1.18"] + go: ["1.18"] steps: - uses: actions/checkout@v3 - name: Setup go @@ -127,7 +127,7 @@ jobs: strategy: fail-fast: false matrix: - go: ["1.16", "1.17", "1.18"] + go: ["1.18"] goos: ["linux"] goarch: ["amd64", "386"] steps: @@ -160,7 +160,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go: ["1.16", "1.17", "1.18"] + go: ["1.18"] goos: ["windows"] goarch: ["amd64"] steps: @@ -209,7 +209,7 @@ jobs: - name: Setup go uses: actions/setup-go@v2 with: - go-version: "1.16" + go-version: "1.18" - uses: actions/cache@v3 with: path: | diff --git a/README.md b/README.md index 7c22b3692..8f54db7b7 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or j ## Requirements -To build Dendrite, you will need Go 1.16 or later. +To build Dendrite, you will need Go 1.18 or later. For a usable federating Dendrite deployment, you will also need: diff --git a/build/scripts/Complement.Dockerfile b/build/scripts/Complement.Dockerfile index 63e3890ee..56877051b 100644 --- a/build/scripts/Complement.Dockerfile +++ b/build/scripts/Complement.Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.16-stretch as build +FROM golang:1.18-stretch as build RUN apt-get update && apt-get install -y sqlite3 WORKDIR /build @@ -27,6 +27,6 @@ EXPOSE 8008 8448 # At runtime, generate TLS cert based on the CA now mounted at /ca # At runtime, replace the SERVER_NAME with what we are told CMD ./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \ - ./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \ - cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \ - ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0} + ./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \ + cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \ + ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0} diff --git a/build/scripts/ComplementLocal.Dockerfile b/build/scripts/ComplementLocal.Dockerfile index a9feb4cd1..704359a28 100644 --- a/build/scripts/ComplementLocal.Dockerfile +++ b/build/scripts/ComplementLocal.Dockerfile @@ -6,7 +6,7 @@ # # Use these mounts to make use of this dockerfile: # COMPLEMENT_HOST_MOUNTS='/your/local/dendrite:/dendrite:ro;/your/go/path:/go:ro' -FROM golang:1.16-stretch +FROM golang:1.18-stretch RUN apt-get update && apt-get install -y sqlite3 WORKDIR /runtime @@ -16,24 +16,24 @@ EXPOSE 8008 8448 # This script compiles Dendrite for us. RUN echo '\ -#!/bin/bash -eux \n\ -if test -f "/runtime/dendrite-monolith-server"; then \n\ + #!/bin/bash -eux \n\ + if test -f "/runtime/dendrite-monolith-server"; then \n\ echo "Skipping compilation; binaries exist" \n\ exit 0 \n\ -fi \n\ -cd /dendrite \n\ -go build -v -o /runtime /dendrite/cmd/dendrite-monolith-server \n\ -' > compile.sh && chmod +x compile.sh + fi \n\ + cd /dendrite \n\ + go build -v -o /runtime /dendrite/cmd/dendrite-monolith-server \n\ + ' > compile.sh && chmod +x compile.sh # This script runs Dendrite for us. Must be run in the /runtime directory. RUN echo '\ -#!/bin/bash -eu \n\ -./generate-keys --private-key matrix_key.pem \n\ -./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key \n\ -./generate-config -server $SERVER_NAME --ci > dendrite.yaml \n\ -cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates \n\ -./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\ -' > run.sh && chmod +x run.sh + #!/bin/bash -eu \n\ + ./generate-keys --private-key matrix_key.pem \n\ + ./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key \n\ + ./generate-config -server $SERVER_NAME --ci > dendrite.yaml \n\ + cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates \n\ + ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\ + ' > run.sh && chmod +x run.sh WORKDIR /cache diff --git a/build/scripts/ComplementPostgres.Dockerfile b/build/scripts/ComplementPostgres.Dockerfile index 4e26faa58..a8b4fbb1d 100644 --- a/build/scripts/ComplementPostgres.Dockerfile +++ b/build/scripts/ComplementPostgres.Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.16-stretch as build +FROM golang:1.18-stretch as build RUN apt-get update && apt-get install -y postgresql WORKDIR /build @@ -9,16 +9,16 @@ RUN sed -i "s%127.0.0.1/32 md5%127.0.0.1/32 trust%g" /etc/ # This entry script starts postgres, waits for it to be up then starts dendrite RUN echo '\ -#!/bin/bash -eu \n\ -pg_lsclusters \n\ -pg_ctlcluster 9.6 main start \n\ - \n\ -until pg_isready \n\ -do \n\ - echo "Waiting for postgres"; \n\ - sleep 1; \n\ -done \n\ -' > run_postgres.sh && chmod +x run_postgres.sh + #!/bin/bash -eu \n\ + pg_lsclusters \n\ + pg_ctlcluster 9.6 main start \n\ + \n\ + until pg_isready \n\ + do \n\ + echo "Waiting for postgres"; \n\ + sleep 1; \n\ + done \n\ + ' > run_postgres.sh && chmod +x run_postgres.sh # we will dump the binaries and config file to this location to ensure any local untracked files # that come from the COPY . . file don't contaminate the build @@ -46,9 +46,9 @@ EXPOSE 8008 8448 # At runtime, generate TLS cert based on the CA now mounted at /ca # At runtime, replace the SERVER_NAME with what we are told CMD /build/run_postgres.sh && ./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \ - ./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \ - # Replace the connection string with a single postgres DB, using user/db = 'postgres' and no password, bump max_conns - sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml && \ - sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml && \ - cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \ - ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0} \ No newline at end of file + ./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \ + # Replace the connection string with a single postgres DB, using user/db = 'postgres' and no password, bump max_conns + sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml && \ + sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml && \ + cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \ + ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0} \ No newline at end of file diff --git a/cmd/dendrite-demo-yggdrasil/README.md b/cmd/dendrite-demo-yggdrasil/README.md index 946333576..14fc3a2db 100644 --- a/cmd/dendrite-demo-yggdrasil/README.md +++ b/cmd/dendrite-demo-yggdrasil/README.md @@ -1,6 +1,6 @@ # Yggdrasil Demo -This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.16 or later. +This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.18 or later. To run the homeserver, start at the root of the Dendrite repository and run: @@ -13,10 +13,10 @@ The following command line arguments are accepted: * `-peer tcp://a.b.c.d:e` to specify a static Yggdrasil peer to connect to - you will need to supply this if you do not have another Yggdrasil node on your network * `-port 12345` to specify a port to listen on for client connections -If you need to find an internet peer, take a look at [this list](https://publicpeers.neilalexander.dev/). +If you need to find an internet peer, take a look at [this list](https://publicpeers.neilalexander.dev/). Then point your favourite Matrix client to the homeserver URL`http://localhost:8008` (or whichever `-port` you specified), create an account and log in. If your peering connection is operational then you should see a `Connected TCP:` line in the log output. If not then try a different peer. -Once logged in, you should be able to open the room directory or join a room by its ID. \ No newline at end of file +Once logged in, you should be able to open the room directory or join a room by its ID. diff --git a/cmd/dendrite-upgrade-tests/main.go b/cmd/dendrite-upgrade-tests/main.go index b7e7da07d..cabd07e70 100644 --- a/cmd/dendrite-upgrade-tests/main.go +++ b/cmd/dendrite-upgrade-tests/main.go @@ -48,7 +48,7 @@ const HEAD = "HEAD" // due to the error: // When using COPY with more than one source file, the destination must be a directory and end with a / // We need to run a postgres anyway, so use the dockerfile associated with Complement instead. -const Dockerfile = `FROM golang:1.16-stretch as build +const Dockerfile = `FROM golang:1.18-stretch as build RUN apt-get update && apt-get install -y postgresql WORKDIR /build diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go index c02140003..6852ab4fb 100644 --- a/cmd/resolve-state/main.go +++ b/cmd/resolve-state/main.go @@ -7,6 +7,7 @@ import ( "sort" "strconv" "strings" + "time" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/roomserver/state" @@ -53,12 +54,10 @@ func main() { fmt.Println("Fetching", len(snapshotNIDs), "snapshot NIDs") - cache, err := caching.NewInMemoryLRUCache(true) - if err != nil { - panic(err) - } - - roomserverDB, err := storage.Open(base, &cfg.RoomServer.Database, cache) + roomserverDB, err := storage.Open( + base, &cfg.RoomServer.Database, + caching.NewRistrettoCache(128*1024*1024, time.Hour, true), + ) if err != nil { panic(err) } diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml index c0134c542..c6050e407 100644 --- a/dendrite-sample.monolith.yaml +++ b/dendrite-sample.monolith.yaml @@ -41,6 +41,25 @@ global: max_idle_conns: 5 conn_max_lifetime: -1 + # Configuration for in-memory caches. Caches can often improve performance by + # keeping frequently accessed items (like events, identifiers etc.) in memory + # rather than having to read them from the database. + cache: + # The estimated maximum size for the global cache in bytes, or in terabytes, + # gigabytes, megabytes or kilobytes when the appropriate 'tb', 'gb', 'mb' or + # 'kb' suffix is specified. Note that this is not a hard limit, nor is it a + # memory limit for the entire process. A cache that is too small may ultimately + # provide little or no benefit. + max_size_estimated: 1gb + + # The maximum amount of time that a cache entry can live for in memory before + # it will be evicted and/or refreshed from the database. Lower values result in + # easier admission of new cache entries but may also increase database load in + # comparison to higher values, so adjust conservatively. Higher values may make + # it harder for new items to make it into the cache, e.g. if new rooms suddenly + # become popular. + max_age: 1h + # The server name to delegate server-server communications to, with optional port # e.g. localhost:443 well_known_server_name: "" diff --git a/dendrite-sample.polylith.yaml b/dendrite-sample.polylith.yaml index b9e3e5e56..ea3d9d689 100644 --- a/dendrite-sample.polylith.yaml +++ b/dendrite-sample.polylith.yaml @@ -31,6 +31,25 @@ global: # considered valid by other homeservers. key_validity_period: 168h0m0s + # Configuration for in-memory caches. Caches can often improve performance by + # keeping frequently accessed items (like events, identifiers etc.) in memory + # rather than having to read them from the database. + cache: + # The estimated maximum size for the global cache in bytes, or in terabytes, + # gigabytes, megabytes or kilobytes when the appropriate 'tb', 'gb', 'mb' or + # 'kb' suffix is specified. Note that this is not a hard limit, nor is it a + # memory limit for the entire process. A cache that is too small may ultimately + # provide little or no benefit. + max_size_estimated: 1gb + + # The maximum amount of time that a cache entry can live for in memory before + # it will be evicted and/or refreshed from the database. Lower values result in + # easier admission of new cache entries but may also increase database load in + # comparison to higher values, so adjust conservatively. Higher values may make + # it harder for new items to make it into the cache, e.g. if new rooms suddenly + # become popular. + max_age: 1h + # The server name to delegate server-server communications to, with optional port # e.g. localhost:443 well_known_server_name: "" diff --git a/docs/installation/1_planning.md b/docs/installation/1_planning.md index 45012d80c..d4f3d7052 100644 --- a/docs/installation/1_planning.md +++ b/docs/installation/1_planning.md @@ -75,7 +75,7 @@ In order to install Dendrite, you will need to satisfy the following dependencie ### Go -At this time, Dendrite supports being built with Go 1.16 or later. We do not support building +At this time, Dendrite supports being built with Go 1.18 or later. We do not support building Dendrite with older versions of Go than this. If you are installing Go using a package manager, you should check (by running `go version`) that you are using a suitable version before you start. diff --git a/federationapi/federationapi_keys_test.go b/federationapi/federationapi_keys_test.go index 73f737717..d1bfe1847 100644 --- a/federationapi/federationapi_keys_test.go +++ b/federationapi/federationapi_keys_test.go @@ -9,7 +9,6 @@ import ( "io/ioutil" "net/http" "os" - "reflect" "testing" "time" @@ -64,10 +63,7 @@ func TestMain(m *testing.M) { } // Create a new cache but don't enable prometheus! - s.cache, err = caching.NewInMemoryLRUCache(false) - if err != nil { - panic("can't create cache: " + err.Error()) - } + s.cache = caching.NewRistrettoCache(8*1024*1024, time.Hour, false) // Create a temporary directory for JetStream. d, err := ioutil.TempDir("./", "jetstream*") @@ -170,72 +166,6 @@ func TestServersRequestOwnKeys(t *testing.T) { } } -func TestCachingBehaviour(t *testing.T) { - // Server A will request Server B's key, which has a validity - // period of an hour from now. We should retrieve the key and - // it should make it into the cache automatically. - - req := gomatrixserverlib.PublicKeyLookupRequest{ - ServerName: serverB.name, - KeyID: serverKeyID, - } - ts := gomatrixserverlib.AsTimestamp(time.Now()) - - res, err := serverA.api.FetchKeys( - context.Background(), - map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp{ - req: ts, - }, - ) - if err != nil { - t.Fatalf("server A failed to retrieve server B key: %s", err) - } - if len(res) != 1 { - t.Fatalf("server B should have returned one key but instead returned %d keys", len(res)) - } - if _, ok := res[req]; !ok { - t.Fatalf("server B isn't included in the key fetch response") - } - - // At this point, if the previous key request was a success, - // then the cache should now contain the key. Check if that's - // the case - if it isn't then there's something wrong with - // the cache implementation or we failed to get the key. - - cres, ok := serverA.cache.GetServerKey(req, ts) - if !ok { - t.Fatalf("server B key should be in cache but isn't") - } - if !reflect.DeepEqual(cres, res[req]) { - t.Fatalf("the cached result from server B wasn't what server B gave us") - } - - // If we ask the cache for the same key but this time for an event - // that happened in +30 minutes. Since the validity period is for - // another hour, then we should get a response back from the cache. - - _, ok = serverA.cache.GetServerKey( - req, - gomatrixserverlib.AsTimestamp(time.Now().Add(time.Minute*30)), - ) - if !ok { - t.Fatalf("server B key isn't in cache when it should be (+30 minutes)") - } - - // If we ask the cache for the same key but this time for an event - // that happened in +90 minutes then we should expect to get no - // cache result. This is because the cache shouldn't return a result - // that is obviously past the validity of the event. - - _, ok = serverA.cache.GetServerKey( - req, - gomatrixserverlib.AsTimestamp(time.Now().Add(time.Minute*90)), - ) - if ok { - t.Fatalf("server B key is in cache when it shouldn't be (+90 minutes)") - } -} - func TestRenewalBehaviour(t *testing.T) { // Server A will request Server C's key but their validity period // is an hour in the past. We'll retrieve the key as, even though it's @@ -262,32 +192,7 @@ func TestRenewalBehaviour(t *testing.T) { t.Fatalf("server C isn't included in the key fetch response") } - // If we ask the cache for the server key for an event that happened - // 90 minutes ago then we should get a cache result, as the key hadn't - // passed its validity by that point. The fact that the key is now in - // the cache is, in itself, proof that we successfully retrieved the - // key before. - - oldcached, ok := serverA.cache.GetServerKey( - req, - gomatrixserverlib.AsTimestamp(time.Now().Add(-time.Minute*90)), - ) - if !ok { - t.Fatalf("server C key isn't in cache when it should be (-90 minutes)") - } - - // If we now ask the cache for the same key but this time for an event - // that only happened 30 minutes ago then we shouldn't get a cached - // result, as the event happened after the key validity expired. This - // is really just for sanity checking. - - _, ok = serverA.cache.GetServerKey( - req, - gomatrixserverlib.AsTimestamp(time.Now().Add(-time.Minute*30)), - ) - if ok { - t.Fatalf("server B key is in cache when it shouldn't be (-30 minutes)") - } + originalValidity := res[req].ValidUntilTS // We're now going to kick server C into renewing its key. Since we're // happy at this point that the key that we already have is from the past @@ -308,24 +213,13 @@ func TestRenewalBehaviour(t *testing.T) { if len(res) != 1 { t.Fatalf("server C should have returned one key but instead returned %d keys", len(res)) } - if _, ok = res[req]; !ok { + if _, ok := res[req]; !ok { t.Fatalf("server C isn't included in the key fetch response") } - // We're now going to ask the cache what the new key validity is. If - // it is still the same as the previous validity then we've failed to - // retrieve the renewed key. If it's newer then we've successfully got - // the renewed key. + currentValidity := res[req].ValidUntilTS - newcached, ok := serverA.cache.GetServerKey( - req, - gomatrixserverlib.AsTimestamp(time.Now().Add(-time.Minute*30)), - ) - if !ok { - t.Fatalf("server B key isn't in cache when it shouldn't be (post-renewal)") + if originalValidity == currentValidity { + t.Fatalf("server C key should have renewed but didn't") } - if oldcached.ValidUntilTS >= newcached.ValidUntilTS { - t.Fatalf("the server B key should have been renewed but wasn't") - } - t.Log(res) } diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go index a5797645e..cde87a0ac 100644 --- a/federationapi/routing/invite.go +++ b/federationapi/routing/invite.go @@ -26,6 +26,7 @@ import ( "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" + "github.com/sirupsen/logrus" ) // InviteV2 implements /_matrix/federation/v2/invite/{roomID}/{eventID} @@ -141,10 +142,17 @@ func processInvite( } // Check that the event is signed by the server sending the request. - redacted := event.Redact() + redacted, err := gomatrixserverlib.RedactEventJSON(event.JSON(), event.Version()) + if err != nil { + logrus.WithError(err).Errorf("XXX: invite.go") + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("The event JSON could not be redacted"), + } + } verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{ ServerName: event.Origin(), - Message: redacted.JSON(), + Message: redacted, AtTS: event.OriginServerTS(), StrictValidityChecking: true, }} diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 8d779c3d7..41004cf51 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -266,10 +266,17 @@ func SendJoin( } // Check that the event is signed by the server sending the request. - redacted := event.Redact() + redacted, err := gomatrixserverlib.RedactEventJSON(event.JSON(), event.Version()) + if err != nil { + logrus.WithError(err).Errorf("XXX: join.go") + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("The event JSON could not be redacted"), + } + } verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{ ServerName: event.Origin(), - Message: redacted.JSON(), + Message: redacted, AtTS: event.OriginServerTS(), StrictValidityChecking: true, }} diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index 54b2c3e84..dbaf68e5b 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -231,10 +231,17 @@ func SendLeave( } // Check that the event is signed by the server sending the request. - redacted := event.Redact() + redacted, err := gomatrixserverlib.RedactEventJSON(event.JSON(), event.Version()) + if err != nil { + logrus.WithError(err).Errorf("XXX: leave.go") + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("The event JSON could not be redacted"), + } + } verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{ ServerName: event.Origin(), - Message: redacted.JSON(), + Message: redacted, AtTS: event.OriginServerTS(), StrictValidityChecking: true, }} diff --git a/go.mod b/go.mod index cf178cfcc..2a2a037c1 100644 --- a/go.mod +++ b/go.mod @@ -7,56 +7,39 @@ replace github.com/nats-io/nats.go => github.com/neilalexander/nats.go v1.13.1-0 require ( github.com/Arceliar/ironwood v0.0.0-20220306165321-319147a02d98 github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979 - github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/DATA-DOG/go-sqlmock v1.5.0 - github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect github.com/MFAshby/stdemuxerhook v1.0.0 github.com/Masterminds/semver/v3 v3.1.1 - github.com/Microsoft/go-winio v0.5.1 // indirect github.com/codeclysm/extract v2.2.0+incompatible - github.com/docker/distribution v2.7.1+incompatible // indirect + github.com/dgraph-io/ristretto v0.1.1-0.20220403145359-8e850b710d6d github.com/docker/docker v20.10.16+incompatible github.com/docker/go-connections v0.4.0 - github.com/docker/go-units v0.4.0 // indirect - github.com/frankban/quicktest v1.14.3 // indirect github.com/getsentry/sentry-go v0.13.0 - github.com/gogo/protobuf v1.3.2 // indirect github.com/gologme/log v1.3.0 github.com/google/go-cmp v0.5.8 github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.5.0 - github.com/h2non/filetype v1.1.3 // indirect - github.com/hashicorp/golang-lru v0.5.4 - github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494 // indirect github.com/kardianos/minwinsvc v1.0.0 github.com/lib/pq v1.10.5 github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20220701090733-da53994b0c7f + github.com/matrix-org/gomatrixserverlib v0.0.0-20220711125303-3bb2e997a44c github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.13 - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect - github.com/miekg/dns v1.1.49 // indirect - github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect - github.com/morikuni/aec v1.0.0 // indirect github.com/nats-io/nats-server/v2 v2.7.4-0.20220309205833-773636c1c5bb github.com/nats-io/nats.go v1.14.0 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79 - github.com/onsi/gomega v1.17.0 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect github.com/opentracing/opentracing-go v1.2.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 github.com/pressly/goose v2.7.0+incompatible github.com/prometheus/client_golang v1.12.2 github.com/sirupsen/logrus v1.8.1 - github.com/stretchr/objx v0.2.0 // indirect github.com/stretchr/testify v1.7.1 github.com/tidwall/gjson v1.14.1 github.com/tidwall/sjson v1.2.4 @@ -68,14 +51,68 @@ require ( golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9 golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd golang.org/x/net v0.0.0-20220524220425-1d687d428aca - golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 - google.golang.org/protobuf v1.27.1 // indirect gopkg.in/h2non/bimg.v1 v1.1.9 gopkg.in/yaml.v2 v2.4.0 - gopkg.in/yaml.v3 v3.0.0 // indirect - gotest.tools/v3 v3.0.3 // indirect nhooyr.io/websocket v1.8.7 ) -go 1.16 +require ( + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect + github.com/Microsoft/go-winio v0.5.1 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cheekybits/genny v1.0.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/docker/distribution v2.7.1+incompatible // indirect + github.com/docker/go-units v0.4.0 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/frankban/quicktest v1.14.3 // indirect + github.com/fsnotify/fsnotify v1.4.9 // indirect + github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/h2non/filetype v1.1.3 // indirect + github.com/juju/errors v0.0.0-20220203013757-bd733f3c86b9 // indirect + github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494 // indirect + github.com/klauspost/compress v1.14.4 // indirect + github.com/lucas-clemente/quic-go v0.26.0 // indirect + github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect + github.com/marten-seemann/qtls-go1-17 v0.1.1 // indirect + github.com/marten-seemann/qtls-go1-18 v0.1.1 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/miekg/dns v1.1.49 // indirect + github.com/minio/highwayhash v1.0.2 // indirect + github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/nats-io/jwt/v2 v2.2.1-0.20220330180145-442af02fd36a // indirect + github.com/nats-io/nkeys v0.3.0 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + github.com/nxadm/tail v1.4.8 // indirect + github.com/onsi/ginkgo v1.16.4 // indirect + github.com/onsi/gomega v1.17.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.32.1 // indirect + github.com/prometheus/procfs v0.7.3 // indirect + github.com/stretchr/objx v0.2.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect + golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect + golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect + golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b // indirect + golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect + golang.org/x/tools v0.1.10 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/protobuf v1.27.1 // indirect + gopkg.in/macaroon.v2 v2.1.0 // indirect + gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect + gopkg.in/yaml.v3 v3.0.0 // indirect + gotest.tools/v3 v3.0.3 // indirect +) + +go 1.18 diff --git a/go.sum b/go.sum index 3ff09b2dc..98549f702 100644 --- a/go.sum +++ b/go.sum @@ -38,7 +38,6 @@ dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBr dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= -github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/Arceliar/ironwood v0.0.0-20211125050254-8951369625d0/go.mod h1:RP72rucOFm5udrnEzTmIWLRVGQiV/fSUAQXJ0RST/nk= github.com/Arceliar/ironwood v0.0.0-20220306165321-319147a02d98 h1:PsaZb47k7WB1V+AlGpb+W7SM+ZOhp16vVevg5gl9YkU= github.com/Arceliar/ironwood v0.0.0-20220306165321-319147a02d98/go.mod h1:RP72rucOFm5udrnEzTmIWLRVGQiV/fSUAQXJ0RST/nk= @@ -49,13 +48,10 @@ github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOEl github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= -github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= -github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/MFAshby/stdemuxerhook v1.0.0 h1:1XFGzakrsHMv76AeanPDL26NOgwjPl/OUxbGhJthwMc= github.com/MFAshby/stdemuxerhook v1.0.0/go.mod h1:nLMI9FUf9Hz98n+yAXsTMUR4RZQy28uCTLG1Fzvj/uY= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= @@ -65,10 +61,8 @@ github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpz github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= github.com/RyanCarrier/dijkstra v1.0.0/go.mod h1:5agGUBNEtUAGIANmbw09fuO3a2htPEkc1jNH01qxCWA= github.com/RyanCarrier/dijkstra-1 v0.0.0-20170512020943-0e5801a26345/go.mod h1:OK4EvWJ441LQqGzed5NGB6vKBAE34n3z7iayPcEwr30= -github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= -github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/albertorestifo/dijkstra v0.0.0-20160910063646-aba76f725f72/go.mod h1:o+JdB7VetTHjLhU0N57x18B9voDBQe0paApdEAEoEfw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -88,8 +82,6 @@ github.com/anacrolix/missinggo v1.2.1/go.mod h1:J5cMhif8jPmFoC3+Uvob3OXXNIhOUikz github.com/anacrolix/missinggo/perf v1.0.0/go.mod h1:ljAFWkBuzkO12MQclXzZrosP5urunoLS0Cbvb4V0uMQ= github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -114,18 +106,15 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/codeclysm/extract v2.2.0+incompatible h1:q3wyckoA30bhUSiwdQezMqVhwd8+WGE64/GL//LtUhI= github.com/codeclysm/extract v2.2.0+incompatible/go.mod h1:2nhFMPHiU9At61hz+12bfrlpXSUrOnK+wR+KlGO4Uks= -github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= +github.com/dgraph-io/ristretto v0.1.1-0.20220403145359-8e850b710d6d h1:Wrc3UKTS+cffkOx0xRGFC+ZesNuTfn0ThvEC72N0krk= +github.com/dgraph-io/ristretto v0.1.1-0.20220403145359-8e850b710d6d/go.mod h1:RAy2GVV4sTWVlNMavv3xhLsk18rxhfhDnombTe6EF5c= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= @@ -137,16 +126,14 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= @@ -157,7 +144,6 @@ github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUork github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= github.com/getsentry/sentry-go v0.13.0 h1:20dgTiUSfxRB/EhMPtxcL9ZEbM1ZdR+W/7f7NWD+xWo= github.com/getsentry/sentry-go v0.13.0/go.mod h1:EOsfu5ZdvKPfeHYV6pTVQnsjfp30+XA7//UooKNumH0= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -165,11 +151,9 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs= -github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= -github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -181,7 +165,6 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= @@ -189,7 +172,6 @@ github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD87 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= -github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= @@ -202,8 +184,8 @@ github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/E github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -239,7 +221,6 @@ github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8l github.com/gologme/log v1.2.0/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U= github.com/gologme/log v1.3.0 h1:l781G4dE+pbigClDSDzSaaYKtiueHCILUa/qSDsmHAo= github.com/gologme/log v1.3.0/go.mod h1:yKT+DvIPdDdDoPtqFrFxheooyVmoqi0BAsw+erN3wA4= -github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -289,24 +270,13 @@ github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hjson/hjson-go v3.1.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk= github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= -github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= -github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk= -github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= -github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -318,67 +288,28 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/juju/ansiterm v0.0.0-20160907234532-b99631de12cf/go.mod h1:UJSiEoRfvx3hP73CvoARgeLjaIOjybY9vj8PUPPFGeU= -github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a/go.mod h1:UJSiEoRfvx3hP73CvoARgeLjaIOjybY9vj8PUPPFGeU= -github.com/juju/clock v0.0.0-20190205081909-9c5c9712527c/go.mod h1:nD0vlnrUjcjJhqN5WuCWZyzfd5AHZAC9/ajvbSx69xA= -github.com/juju/cmd v0.0.0-20171107070456-e74f39857ca0/go.mod h1:yWJQHl73rdSX4DHVKGqkAip+huBslxRwS8m9CrOLq18= -github.com/juju/collections v0.0.0-20200605021417-0d0ec82b7271/go.mod h1:5XgO71dV1JClcOJE+4dzdn4HrI5LiyKd7PlVG6eZYhY= -github.com/juju/errors v0.0.0-20150916125642-1b5e39b83d18/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= -github.com/juju/errors v0.0.0-20200330140219-3fe23663418f/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= github.com/juju/errors v0.0.0-20220203013757-bd733f3c86b9 h1:EJHbsNpQyupmMeWTq7inn+5L/WZ7JfzCVPJ+DP9McCQ= github.com/juju/errors v0.0.0-20220203013757-bd733f3c86b9/go.mod h1:TRm7EVGA3mQOqSVcBySRY7a9Y1/gyVhh/WTCnc5sD4U= -github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= -github.com/juju/httpprof v0.0.0-20141217160036-14bf14c30767/go.mod h1:+MaLYz4PumRkkyHYeXJ2G5g5cIW0sli2bOfpmbaMV/g= -github.com/juju/loggo v0.0.0-20170605014607-8232ab8918d9/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= -github.com/juju/loggo v0.0.0-20200526014432-9ce3a2e09b5e/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/juju/loggo v0.0.0-20210728185423-eebad3a902c4 h1:NO5tuyw++EGLnz56Q8KMyDZRwJwWO8jQnj285J3FOmY= -github.com/juju/loggo v0.0.0-20210728185423-eebad3a902c4/go.mod h1:NIXFioti1SmKAlKNuUwbMenNdef59IF52+ZzuOmHYkg= github.com/juju/mgo/v2 v2.0.0-20210302023703-70d5d206e208 h1:/WiCm+Vpj87e4QWuWwPD/bNE9kDrWCLvPBHOQNcG2+A= -github.com/juju/mgo/v2 v2.0.0-20210302023703-70d5d206e208/go.mod h1:0OChplkvPTZ174D2FYZXg4IB9hbEwyHkD+zT+/eK+Fg= -github.com/juju/mutex v0.0.0-20171110020013-1fe2a4bf0a3a/go.mod h1:Y3oOzHH8CQ0Ppt0oCKJ2JFO81/EsWenH5AEqigLH+yY= -github.com/juju/retry v0.0.0-20151029024821-62c620325291/go.mod h1:OohPQGsr4pnxwD5YljhQ+TZnuVRYpa5irjugL1Yuif4= -github.com/juju/retry v0.0.0-20180821225755-9058e192b216/go.mod h1:OohPQGsr4pnxwD5YljhQ+TZnuVRYpa5irjugL1Yuif4= -github.com/juju/testing v0.0.0-20180402130637-44801989f0f7/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= -github.com/juju/testing v0.0.0-20190723135506-ce30eb24acd2/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= -github.com/juju/testing v0.0.0-20210302031854-2c7ee8570c07/go.mod h1:7lxZW0B50+xdGFkvhAb8bwAGt6IU87JB1H9w4t8MNVM= -github.com/juju/testing v0.0.0-20220202055744-1ad0816210a6/go.mod h1:QgWc2UdIPJ8t3rnvv95tFNOsQDfpXYEZDbP281o3b2c= github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494 h1:XEDzpuZb8Ma7vLja3+5hzUqVTvAqm5Y+ygvnDs5iTMM= github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494/go.mod h1:rUquetT0ALL48LHZhyRGvjjBH8xZaZ8dFClulKK5wK4= -github.com/juju/utils v0.0.0-20180424094159-2000ea4ff043/go.mod h1:6/KLg8Wz/y2KVGWEpkK9vMNGkOnu4k/cqs8Z1fKjTOk= -github.com/juju/utils v0.0.0-20200116185830-d40c2fe10647/go.mod h1:6/KLg8Wz/y2KVGWEpkK9vMNGkOnu4k/cqs8Z1fKjTOk= -github.com/juju/utils/v2 v2.0.0-20200923005554-4646bfea2ef1/go.mod h1:fdlDtQlzundleLLz/ggoYinEt/LmnrpNKcNTABQATNI= -github.com/juju/utils/v3 v3.0.0-20220130232349-cd7ecef0e94a/go.mod h1:LzwbbEN7buYjySp4nqnti6c6olSqRXUk6RkbSUUP1n8= -github.com/juju/version v0.0.0-20161031051906-1f41e27e54f2/go.mod h1:kE8gK5X0CImdr7qpSKl3xB2PmpySSmfj7zVbkZFs81U= -github.com/juju/version v0.0.0-20180108022336-b64dbd566305/go.mod h1:kE8gK5X0CImdr7qpSKl3xB2PmpySSmfj7zVbkZFs81U= -github.com/juju/version v0.0.0-20191219164919-81c1be00b9a6/go.mod h1:kE8gK5X0CImdr7qpSKl3xB2PmpySSmfj7zVbkZFs81U= -github.com/juju/version/v2 v2.0.0-20211007103408-2e8da085dc23/go.mod h1:Ljlbryh9sYaUSGXucslAEDf0A2XUSGvDbHJgW8ps6nc= -github.com/julienschmidt/httprouter v1.1.1-0.20151013225520-77a895ad01eb/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kardianos/minwinsvc v1.0.0 h1:+JfAi8IBJna0jY2dJGZqi7o15z13JelFIklJCAENALA= github.com/kardianos/minwinsvc v1.0.0/go.mod h1:Bgd0oc+D0Qo3bBytmNtyRKVlp85dAloLKhfxanPFFRc= -github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8= -github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE= -github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE= -github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro= -github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.14.4 h1:eijASRJcobkVtSt81Olfh7JX43osYLwy5krOJo6YEu4= github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -386,19 +317,15 @@ github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y= -github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v1.10.5 h1:J+gdV2cUmX7ZqL2B0lFcW0m+egaHC2V3lpO8nWxyYiQ= github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lucas-clemente/quic-go v0.26.0 h1:ALBQXr9UJ8A1LyzvceX4jd9QFsHvlI0RR6BkV16o00A= github.com/lucas-clemente/quic-go v0.26.0/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI= -github.com/lunixbochs/vtclean v0.0.0-20160125035106-4fbf7632a2c6/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lxn/walk v0.0.0-20210112085537-c389da54e794/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ= github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= github.com/marten-seemann/qtls-go1-16 v0.1.5 h1:o9JrYPPco/Nukd/HpOHMHZoBDXQqoNtUCmny98/1uqQ= @@ -407,10 +334,6 @@ github.com/marten-seemann/qtls-go1-17 v0.1.1 h1:DQjHPq+aOzUeh9/lixAGunn6rIOQyWCh github.com/marten-seemann/qtls-go1-17 v0.1.1/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s= github.com/marten-seemann/qtls-go1-18 v0.1.1 h1:qp7p7XXUFL7fpBvSS1sWD+uSqPvzNQK43DH+/qEkj0Y= github.com/marten-seemann/qtls-go1-18 v0.1.1/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= -github.com/masterzen/azure-sdk-for-go v3.2.0-beta.0.20161014135628-ee4f0065d00c+incompatible/go.mod h1:mf8fjOu33zCqxUjuiU3I8S1lJMyEAlH+0F2+M5xl3hE= -github.com/masterzen/simplexml v0.0.0-20160608183007-4572e39b1ab9/go.mod h1:kCEbxUJlNDEBNbdQMkPSp6yaKcRXVI6f4ddk8Riv4bc= -github.com/masterzen/winrm v0.0.0-20161014151040-7a535cd943fc/go.mod h1:CfZSN7zwz5gJiFhZJz49Uzk7mEBHIceWmbFmYx7Hf7E= -github.com/masterzen/xmlpath v0.0.0-20140218185901-13f4951698ad/go.mod h1:A0zPC53iKKKcXYxr4ROjpQRQ5FgJXtelNdSmHHuq/tY= github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e h1:DP5RC0Z3XdyBEW5dKt8YPeN6vZbm6OzVaGVp7f1BQRM= github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw2QV3YD/fRrzEDPNGgTlJlvXY0EHHnT87wF3OA= @@ -418,43 +341,31 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220701090733-da53994b0c7f h1:XF2+J6sOq07yhK1I7ItwsgRwXorjj7gqiCvgZ4dn8W8= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220701090733-da53994b0c7f/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220711125303-3bb2e997a44c h1:mt30TDK8kXKV+nCmVfnqoXsh842N+74kvZw7DXuS/JQ= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220711125303-3bb2e997a44c/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk= github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a h1:DdG8vXMlZ65EAtc4V+3t7zHZ2Gqs24pSnyXS+4BRHUs= github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= -github.com/mattn/go-colorable v0.0.6/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-isatty v0.0.0-20160806122752-66b8e73f3f5c/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.13 h1:1tj15ngiFfcZzii7yd82foL+ks+ouQcj8j/TPq3fk1I= github.com/mattn/go-sqlite3 v1.14.13/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/mattomatic/dijkstra v0.0.0-20130617153013-6f6d134eb237/go.mod h1:UOnLAUmVG5paym8pD3C4B9BQylUDC2vXFJJpT7JrlEA= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= -github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/miekg/dns v1.1.25/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.49 h1:qe0mQU3Z/XpFeE+AEBo2rqaS1IPBJ3anmqZ4XiZJVG8= github.com/miekg/dns v1.1.49/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 h1:yH0SvLzcbZxcJXho2yh7CqdENGMQe73Cw3woZBpPli0= github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= @@ -467,7 +378,6 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -491,7 +401,6 @@ github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79 h1:Dmx8g2747UTVPzSkmohk84S3g/uWqd6+f4SSLPhLcfA= github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79/go.mod h1:E26fwEtRNigBfFfHDWsklmo0T7Ixbg0XXgck+Hq4O9k= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= @@ -515,10 +424,8 @@ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYr github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= -github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -558,9 +465,7 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= -github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= @@ -582,7 +487,6 @@ github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ= github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk= github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -593,15 +497,9 @@ github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= @@ -631,30 +529,16 @@ github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVK github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= -github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= -github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/yggdrasil-network/yggdrasil-go v0.4.3 h1:LNS7kNpKzFlxQ9xmD5tfmMEvzwa+utBoD6pV9t2a8q4= github.com/yggdrasil-network/yggdrasil-go v0.4.3/go.mod h1:A1/8kOQT7vzBxlkQtLf1KzJR0cbfL/2zjOCiYOAdjjo= -github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= -github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -672,26 +556,20 @@ go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= -golang.org/x/crypto v0.0.0-20180214000028-650f4a345ab4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -736,21 +614,19 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180406214816-61147c48b25b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -759,7 +635,6 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -776,7 +651,6 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= @@ -785,10 +659,8 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210927181540-4e4d966f7476/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211011170408-caeb26a5c8c0/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211101193420-4a448f8816b3/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220524220425-1d687d428aca h1:xTaFYiPROfpPhqrfTIDXj0ri1SpfueYT951s4bAuDO8= golang.org/x/net v0.0.0-20220524220425-1d687d428aca/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -816,10 +688,8 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -829,9 +699,7 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -879,14 +747,11 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211102192858-4dd72447c267/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -895,29 +760,24 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b h1:NXqSWXSRUSCaFuvitrWtU169I3876zRTalMRbfd6LL0= golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b/go.mod h1:EFNZuWvGYxIRUEX+K8UmCFwYmZjqcrnq15ZuVldZkZ0= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 h1:GZokNIeuVkl3aZHJchRrr13WCsols02MLUcz1U9is6M= golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -960,8 +820,9 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= -golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098 h1:YuekqPskqwCCPM79F1X5Dhv4ezTCj+Ki1oNwiafxkA0= golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= +golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1066,32 +927,22 @@ google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+Rur google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20160105164936-4f90aeace3a2/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v1 v1.0.0-20161222125816-442357a80af5/go.mod h1:u0ALmqvLRxLI95fkdCEWrE6mhWYZW1aMOJHp5YXLHTg= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/h2non/bimg.v1 v1.1.9 h1:wZIUbeOnwr37Ta4aofhIv8OI8v4ujpjXC9mXnAGpQjM= gopkg.in/h2non/bimg.v1 v1.1.9/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= gopkg.in/h2non/gock.v1 v1.0.14 h1:fTeu9fcUvSnLNacYvYI54h+1/XEteDyHvrVCZEEEYNM= gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= -gopkg.in/httprequest.v1 v1.1.1/go.mod h1:/CkavNL+g3qLOrpFHVrEx4NKepeqR4XTZWNj4sGGjz0= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/macaroon.v2 v2.1.0 h1:HZcsjBCzq9t0eBPMKqTN/uSN6JOm78ZJ2INbqcBQOUI= gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o= -gopkg.in/mgo.v2 v2.0.0-20160818015218-f2b6f6c918c4/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637/go.mod h1:BHsqpu/nsuzkT5BpiH1EMZPLyqSMM8JbIavyFACoFNk= -gopkg.in/yaml.v2 v2.0.0-20170712054546-1be3d31502d6/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1100,7 +951,6 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= @@ -1117,8 +967,6 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM= -launchpad.net/xmlpath v0.0.0-20130614043138-000000000004/go.mod h1:vqyExLOM3qBx7mvYRkoxjSCF945s0mbe7YynlKYXtsA= nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/internal/caching/cache_federationevents.go b/internal/caching/cache_federationevents.go index b79cc809f..24af51bdc 100644 --- a/internal/caching/cache_federationevents.go +++ b/internal/caching/cache_federationevents.go @@ -1,18 +1,9 @@ package caching import ( - "fmt" - "github.com/matrix-org/gomatrixserverlib" ) -const ( - FederationEventCacheName = "federation_event" - FederationEventCacheMaxEntries = 256 - FederationEventCacheMutable = true // to allow use of Unset only - FederationEventCacheMaxAge = CacheNoMaxAge -) - // FederationCache contains the subset of functions needed for // a federation event cache. type FederationCache interface { @@ -26,43 +17,25 @@ type FederationCache interface { } func (c Caches) GetFederationQueuedPDU(eventNID int64) (*gomatrixserverlib.HeaderedEvent, bool) { - key := fmt.Sprintf("%d", eventNID) - val, found := c.FederationEvents.Get(key) - if found && val != nil { - if event, ok := val.(*gomatrixserverlib.HeaderedEvent); ok { - return event, true - } - } - return nil, false + return c.FederationPDUs.Get(eventNID) } func (c Caches) StoreFederationQueuedPDU(eventNID int64, event *gomatrixserverlib.HeaderedEvent) { - key := fmt.Sprintf("%d", eventNID) - c.FederationEvents.Set(key, event) + c.FederationPDUs.Set(eventNID, event) } func (c Caches) EvictFederationQueuedPDU(eventNID int64) { - key := fmt.Sprintf("%d", eventNID) - c.FederationEvents.Unset(key) + c.FederationPDUs.Unset(eventNID) } func (c Caches) GetFederationQueuedEDU(eventNID int64) (*gomatrixserverlib.EDU, bool) { - key := fmt.Sprintf("%d", eventNID) - val, found := c.FederationEvents.Get(key) - if found && val != nil { - if event, ok := val.(*gomatrixserverlib.EDU); ok { - return event, true - } - } - return nil, false + return c.FederationEDUs.Get(eventNID) } func (c Caches) StoreFederationQueuedEDU(eventNID int64, event *gomatrixserverlib.EDU) { - key := fmt.Sprintf("%d", eventNID) - c.FederationEvents.Set(key, event) + c.FederationEDUs.Set(eventNID, event) } func (c Caches) EvictFederationQueuedEDU(eventNID int64) { - key := fmt.Sprintf("%d", eventNID) - c.FederationEvents.Unset(key) + c.FederationEDUs.Unset(eventNID) } diff --git a/internal/caching/cache_lazy_load_members.go b/internal/caching/cache_lazy_load_members.go index f0d495065..0d7009c94 100644 --- a/internal/caching/cache_lazy_load_members.go +++ b/internal/caching/cache_lazy_load_members.go @@ -1,67 +1,35 @@ package caching import ( - "fmt" - "time" - userapi "github.com/matrix-org/dendrite/userapi/api" ) -const ( - LazyLoadCacheName = "lazy_load_members" - LazyLoadCacheMaxEntries = 128 - LazyLoadCacheMaxUserEntries = 128 - LazyLoadCacheMutable = true - LazyLoadCacheMaxAge = time.Minute * 30 -) +type lazyLoadingCacheKey struct { + UserID string // the user we're querying on behalf of + DeviceID string // the user we're querying on behalf of + RoomID string // the room in question + TargetUserID string // the user whose membership we're asking about +} type LazyLoadCache interface { StoreLazyLoadedUser(device *userapi.Device, roomID, userID, eventID string) IsLazyLoadedUserCached(device *userapi.Device, roomID, userID string) (string, bool) } -func (c Caches) lazyLoadCacheForUser(device *userapi.Device) (*InMemoryLRUCachePartition, error) { - cacheName := fmt.Sprintf("%s/%s", device.UserID, device.ID) - userCache, ok := c.LazyLoading.Get(cacheName) - if ok && userCache != nil { - if cache, ok := userCache.(*InMemoryLRUCachePartition); ok { - return cache, nil - } - } - cache, err := NewInMemoryLRUCachePartition( - LazyLoadCacheName, - LazyLoadCacheMutable, - LazyLoadCacheMaxUserEntries, - LazyLoadCacheMaxAge, - false, - ) - if err != nil { - return nil, err - } - c.LazyLoading.Set(cacheName, cache) - go cacheCleaner(cache) - return cache, nil -} - func (c Caches) StoreLazyLoadedUser(device *userapi.Device, roomID, userID, eventID string) { - cache, err := c.lazyLoadCacheForUser(device) - if err != nil { - return - } - cacheKey := fmt.Sprintf("%s/%s/%s/%s", device.UserID, device.ID, roomID, userID) - cache.Set(cacheKey, eventID) + c.LazyLoading.Set(lazyLoadingCacheKey{ + UserID: device.UserID, + DeviceID: device.ID, + RoomID: roomID, + TargetUserID: userID, + }, eventID) } func (c Caches) IsLazyLoadedUserCached(device *userapi.Device, roomID, userID string) (string, bool) { - cache, err := c.lazyLoadCacheForUser(device) - if err != nil { - return "", false - } - - cacheKey := fmt.Sprintf("%s/%s/%s/%s", device.UserID, device.ID, roomID, userID) - val, ok := cache.Get(cacheKey) - if !ok { - return "", ok - } - return val.(string), ok + return c.LazyLoading.Get(lazyLoadingCacheKey{ + UserID: device.UserID, + DeviceID: device.ID, + RoomID: roomID, + TargetUserID: userID, + }) } diff --git a/internal/caching/cache_roomevents.go b/internal/caching/cache_roomevents.go new file mode 100644 index 000000000..9d5d3b912 --- /dev/null +++ b/internal/caching/cache_roomevents.go @@ -0,0 +1,21 @@ +package caching + +import ( + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" +) + +// RoomServerEventsCache contains the subset of functions needed for +// a roomserver event cache. +type RoomServerEventsCache interface { + GetRoomServerEvent(eventNID types.EventNID) (*gomatrixserverlib.Event, bool) + StoreRoomServerEvent(eventNID types.EventNID, event *gomatrixserverlib.Event) +} + +func (c Caches) GetRoomServerEvent(eventNID types.EventNID) (*gomatrixserverlib.Event, bool) { + return c.RoomServerEvents.Get(int64(eventNID)) +} + +func (c Caches) StoreRoomServerEvent(eventNID types.EventNID, event *gomatrixserverlib.Event) { + c.RoomServerEvents.Set(int64(eventNID), event) +} diff --git a/internal/caching/cache_roominfo.go b/internal/caching/cache_roominfo.go index 60d221285..d03a61077 100644 --- a/internal/caching/cache_roominfo.go +++ b/internal/caching/cache_roominfo.go @@ -1,8 +1,6 @@ package caching import ( - "time" - "github.com/matrix-org/dendrite/roomserver/types" ) @@ -14,13 +12,6 @@ import ( // used from other components as we currently have no way to invalidate // the cache in downstream components. -const ( - RoomInfoCacheName = "roominfo" - RoomInfoCacheMaxEntries = 1024 - RoomInfoCacheMutable = true - RoomInfoCacheMaxAge = time.Minute * 5 -) - // RoomInfosCache contains the subset of functions needed for // a room Info cache. It must only be used from the roomserver only // It is not safe for use from other components. @@ -32,13 +23,7 @@ type RoomInfoCache interface { // GetRoomInfo must only be called from the roomserver only. It is not // safe for use from other components. func (c Caches) GetRoomInfo(roomID string) (types.RoomInfo, bool) { - val, found := c.RoomInfos.Get(roomID) - if found && val != nil { - if roomInfo, ok := val.(types.RoomInfo); ok { - return roomInfo, true - } - } - return types.RoomInfo{}, false + return c.RoomInfos.Get(roomID) } // StoreRoomInfo must only be called from the roomserver only. It is not diff --git a/internal/caching/cache_roomservernids.go b/internal/caching/cache_roomservernids.go index 1918a2f1e..b409aeef2 100644 --- a/internal/caching/cache_roomservernids.go +++ b/internal/caching/cache_roomservernids.go @@ -1,22 +1,14 @@ package caching import ( - "strconv" - "github.com/matrix-org/dendrite/roomserver/types" ) -const ( - RoomServerRoomIDsCacheName = "roomserver_room_ids" - RoomServerRoomIDsCacheMaxEntries = 1024 - RoomServerRoomIDsCacheMutable = false - RoomServerRoomIDsCacheMaxAge = CacheNoMaxAge -) - type RoomServerCaches interface { RoomServerNIDsCache RoomVersionCache RoomInfoCache + RoomServerEventsCache } // RoomServerNIDsCache contains the subset of functions needed for @@ -27,15 +19,9 @@ type RoomServerNIDsCache interface { } func (c Caches) GetRoomServerRoomID(roomNID types.RoomNID) (string, bool) { - val, found := c.RoomServerRoomIDs.Get(strconv.Itoa(int(roomNID))) - if found && val != nil { - if roomID, ok := val.(string); ok { - return roomID, true - } - } - return "", false + return c.RoomServerRoomIDs.Get(int64(roomNID)) } func (c Caches) StoreRoomServerRoomID(roomNID types.RoomNID, roomID string) { - c.RoomServerRoomIDs.Set(strconv.Itoa(int(roomNID)), roomID) + c.RoomServerRoomIDs.Set(int64(roomNID), roomID) } diff --git a/internal/caching/cache_roomversions.go b/internal/caching/cache_roomversions.go index 92d2eab08..afc3d36da 100644 --- a/internal/caching/cache_roomversions.go +++ b/internal/caching/cache_roomversions.go @@ -2,13 +2,6 @@ package caching import "github.com/matrix-org/gomatrixserverlib" -const ( - RoomVersionCacheName = "room_versions" - RoomVersionCacheMaxEntries = 1024 - RoomVersionCacheMutable = false - RoomVersionCacheMaxAge = CacheNoMaxAge -) - // RoomVersionsCache contains the subset of functions needed for // a room version cache. type RoomVersionCache interface { @@ -17,13 +10,7 @@ type RoomVersionCache interface { } func (c Caches) GetRoomVersion(roomID string) (gomatrixserverlib.RoomVersion, bool) { - val, found := c.RoomVersions.Get(roomID) - if found && val != nil { - if roomVersion, ok := val.(gomatrixserverlib.RoomVersion); ok { - return roomVersion, true - } - } - return "", false + return c.RoomVersions.Get(roomID) } func (c Caches) StoreRoomVersion(roomID string, roomVersion gomatrixserverlib.RoomVersion) { diff --git a/internal/caching/cache_serverkeys.go b/internal/caching/cache_serverkeys.go index 4eb10fe6f..cffa101d5 100644 --- a/internal/caching/cache_serverkeys.go +++ b/internal/caching/cache_serverkeys.go @@ -6,13 +6,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -const ( - ServerKeyCacheName = "server_key" - ServerKeyCacheMaxEntries = 4096 - ServerKeyCacheMutable = true - ServerKeyCacheMaxAge = CacheNoMaxAge -) - // ServerKeyCache contains the subset of functions needed for // a server key cache. type ServerKeyCache interface { @@ -34,18 +27,13 @@ func (c Caches) GetServerKey( ) (gomatrixserverlib.PublicKeyLookupResult, bool) { key := fmt.Sprintf("%s/%s", request.ServerName, request.KeyID) val, found := c.ServerKeys.Get(key) - if found && val != nil { - if keyLookupResult, ok := val.(gomatrixserverlib.PublicKeyLookupResult); ok { - if !keyLookupResult.WasValidAt(timestamp, true) { - // The key wasn't valid at the requested timestamp so don't - // return it. The caller will have to work out what to do. - c.ServerKeys.Unset(key) - return gomatrixserverlib.PublicKeyLookupResult{}, false - } - return keyLookupResult, true - } + if found && !val.WasValidAt(timestamp, true) { + // The key wasn't valid at the requested timestamp so don't + // return it. The caller will have to work out what to do. + c.ServerKeys.Unset(key) + return gomatrixserverlib.PublicKeyLookupResult{}, false } - return gomatrixserverlib.PublicKeyLookupResult{}, false + return val, found } func (c Caches) StoreServerKey( diff --git a/internal/caching/cache_space_rooms.go b/internal/caching/cache_space_rooms.go index 6d56cce5f..697f99269 100644 --- a/internal/caching/cache_space_rooms.go +++ b/internal/caching/cache_space_rooms.go @@ -1,31 +1,16 @@ package caching import ( - "time" - "github.com/matrix-org/gomatrixserverlib" ) -const ( - SpaceSummaryRoomsCacheName = "space_summary_rooms" - SpaceSummaryRoomsCacheMaxEntries = 100 - SpaceSummaryRoomsCacheMutable = true - SpaceSummaryRoomsCacheMaxAge = time.Minute * 5 -) - type SpaceSummaryRoomsCache interface { GetSpaceSummary(roomID string) (r gomatrixserverlib.MSC2946SpacesResponse, ok bool) StoreSpaceSummary(roomID string, r gomatrixserverlib.MSC2946SpacesResponse) } func (c Caches) GetSpaceSummary(roomID string) (r gomatrixserverlib.MSC2946SpacesResponse, ok bool) { - val, found := c.SpaceSummaryRooms.Get(roomID) - if found && val != nil { - if resp, ok := val.(gomatrixserverlib.MSC2946SpacesResponse); ok { - return resp, true - } - } - return r, false + return c.SpaceSummaryRooms.Get(roomID) } func (c Caches) StoreSpaceSummary(roomID string, r gomatrixserverlib.MSC2946SpacesResponse) { diff --git a/internal/caching/caches.go b/internal/caching/caches.go index 173e47e5b..14b232dd0 100644 --- a/internal/caching/caches.go +++ b/internal/caching/caches.go @@ -1,28 +1,52 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package caching import ( - "time" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" ) // Caches contains a set of references to caches. They may be // different implementations as long as they satisfy the Cache // interface. type Caches struct { - RoomVersions Cache // RoomVersionCache - ServerKeys Cache // ServerKeyCache - RoomServerRoomNIDs Cache // RoomServerNIDsCache - RoomServerRoomIDs Cache // RoomServerNIDsCache - RoomInfos Cache // RoomInfoCache - FederationEvents Cache // FederationEventsCache - SpaceSummaryRooms Cache // SpaceSummaryRoomsCache - LazyLoading Cache // LazyLoadCache + RoomVersions Cache[string, gomatrixserverlib.RoomVersion] // room ID -> room version + ServerKeys Cache[string, gomatrixserverlib.PublicKeyLookupResult] // server name -> server keys + RoomServerRoomNIDs Cache[string, types.RoomNID] // room ID -> room NID + RoomServerRoomIDs Cache[int64, string] // room NID -> room ID + RoomServerEvents Cache[int64, *gomatrixserverlib.Event] // event NID -> event + RoomInfos Cache[string, types.RoomInfo] // room ID -> room info + FederationPDUs Cache[int64, *gomatrixserverlib.HeaderedEvent] // queue NID -> PDU + FederationEDUs Cache[int64, *gomatrixserverlib.EDU] // queue NID -> EDU + SpaceSummaryRooms Cache[string, gomatrixserverlib.MSC2946SpacesResponse] // room ID -> space response + LazyLoading Cache[lazyLoadingCacheKey, string] // composite key -> event ID } // Cache is the interface that an implementation must satisfy. -type Cache interface { - Get(key string) (value interface{}, ok bool) - Set(key string, value interface{}) - Unset(key string) +type Cache[K keyable, T any] interface { + Get(key K) (value T, ok bool) + Set(key K, value T) + Unset(key K) } -const CacheNoMaxAge = time.Duration(0) +type keyable interface { + // from https://github.com/dgraph-io/ristretto/blob/8e850b710d6df0383c375ec6a7beae4ce48fc8d5/z/z.go#L34 + uint64 | string | []byte | byte | int | int32 | uint32 | int64 | lazyLoadingCacheKey +} + +type costable interface { + CacheCost() int +} diff --git a/internal/caching/impl_inmemorylru.go b/internal/caching/impl_inmemorylru.go deleted file mode 100644 index 594760892..000000000 --- a/internal/caching/impl_inmemorylru.go +++ /dev/null @@ -1,189 +0,0 @@ -package caching - -import ( - "fmt" - "time" - - lru "github.com/hashicorp/golang-lru" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" -) - -func NewInMemoryLRUCache(enablePrometheus bool) (*Caches, error) { - roomVersions, err := NewInMemoryLRUCachePartition( - RoomVersionCacheName, - RoomVersionCacheMutable, - RoomVersionCacheMaxEntries, - RoomVersionCacheMaxAge, - enablePrometheus, - ) - if err != nil { - return nil, err - } - serverKeys, err := NewInMemoryLRUCachePartition( - ServerKeyCacheName, - ServerKeyCacheMutable, - ServerKeyCacheMaxEntries, - ServerKeyCacheMaxAge, - enablePrometheus, - ) - if err != nil { - return nil, err - } - roomServerRoomIDs, err := NewInMemoryLRUCachePartition( - RoomServerRoomIDsCacheName, - RoomServerRoomIDsCacheMutable, - RoomServerRoomIDsCacheMaxEntries, - RoomServerRoomIDsCacheMaxAge, - enablePrometheus, - ) - if err != nil { - return nil, err - } - roomInfos, err := NewInMemoryLRUCachePartition( - RoomInfoCacheName, - RoomInfoCacheMutable, - RoomInfoCacheMaxEntries, - RoomInfoCacheMaxAge, - enablePrometheus, - ) - if err != nil { - return nil, err - } - federationEvents, err := NewInMemoryLRUCachePartition( - FederationEventCacheName, - FederationEventCacheMutable, - FederationEventCacheMaxEntries, - FederationEventCacheMaxAge, - enablePrometheus, - ) - if err != nil { - return nil, err - } - spaceRooms, err := NewInMemoryLRUCachePartition( - SpaceSummaryRoomsCacheName, - SpaceSummaryRoomsCacheMutable, - SpaceSummaryRoomsCacheMaxEntries, - SpaceSummaryRoomsCacheMaxAge, - enablePrometheus, - ) - if err != nil { - return nil, err - } - - lazyLoadCache, err := NewInMemoryLRUCachePartition( - LazyLoadCacheName, - LazyLoadCacheMutable, - LazyLoadCacheMaxEntries, - LazyLoadCacheMaxAge, - enablePrometheus, - ) - if err != nil { - return nil, err - } - - go cacheCleaner( - roomVersions, serverKeys, roomServerRoomIDs, - roomInfos, federationEvents, spaceRooms, lazyLoadCache, - ) - return &Caches{ - RoomVersions: roomVersions, - ServerKeys: serverKeys, - RoomServerRoomIDs: roomServerRoomIDs, - RoomInfos: roomInfos, - FederationEvents: federationEvents, - SpaceSummaryRooms: spaceRooms, - LazyLoading: lazyLoadCache, - }, nil -} - -func cacheCleaner(caches ...*InMemoryLRUCachePartition) { - for { - time.Sleep(time.Minute) - for _, cache := range caches { - // Hold onto the last 10% of the cache entries, since - // otherwise a quiet period might cause us to evict all - // cache entries entirely. - if cache.lru.Len() > cache.maxEntries/10 { - cache.lru.RemoveOldest() - } - } - } -} - -type InMemoryLRUCachePartition struct { - name string - mutable bool - maxEntries int - maxAge time.Duration - lru *lru.Cache -} - -type inMemoryLRUCacheEntry struct { - value interface{} - created time.Time -} - -func NewInMemoryLRUCachePartition(name string, mutable bool, maxEntries int, maxAge time.Duration, enablePrometheus bool) (*InMemoryLRUCachePartition, error) { - var err error - cache := InMemoryLRUCachePartition{ - name: name, - mutable: mutable, - maxEntries: maxEntries, - maxAge: maxAge, - } - cache.lru, err = lru.New(maxEntries) - if err != nil { - return nil, err - } - if enablePrometheus { - promauto.NewGaugeFunc(prometheus.GaugeOpts{ - Namespace: "dendrite", - Subsystem: "caching_in_memory_lru", - Name: name, - }, func() float64 { - return float64(cache.lru.Len()) - }) - } - return &cache, nil -} - -func (c *InMemoryLRUCachePartition) Set(key string, value interface{}) { - if !c.mutable { - if peek, ok := c.lru.Peek(key); ok { - if entry, ok := peek.(*inMemoryLRUCacheEntry); ok && entry.value != value { - panic(fmt.Sprintf("invalid use of immutable cache tries to mutate existing value of %q", key)) - } - } - } - c.lru.Add(key, &inMemoryLRUCacheEntry{ - value: value, - created: time.Now(), - }) -} - -func (c *InMemoryLRUCachePartition) Unset(key string) { - if !c.mutable { - panic(fmt.Sprintf("invalid use of immutable cache tries to unset value of %q", key)) - } - c.lru.Remove(key) -} - -func (c *InMemoryLRUCachePartition) Get(key string) (value interface{}, ok bool) { - v, ok := c.lru.Get(key) - if !ok { - return nil, false - } - entry, ok := v.(*inMemoryLRUCacheEntry) - switch { - case ok && c.maxAge == CacheNoMaxAge: - return entry.value, ok // There's no maximum age policy - case ok && time.Since(entry.created) < c.maxAge: - return entry.value, ok // The value for the key isn't stale - default: - // Either the key was found and it was stale, or the key - // wasn't found at all - c.lru.Remove(key) - return nil, false - } -} diff --git a/internal/caching/impl_ristretto.go b/internal/caching/impl_ristretto.go new file mode 100644 index 000000000..6d625b552 --- /dev/null +++ b/internal/caching/impl_ristretto.go @@ -0,0 +1,200 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package caching + +import ( + "fmt" + "reflect" + "time" + "unsafe" + + "github.com/dgraph-io/ristretto" + "github.com/dgraph-io/ristretto/z" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/gomatrixserverlib" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +const ( + roomVersionsCache byte = iota + 1 + serverKeysCache + roomNIDsCache + roomIDsCache + roomEventsCache + roomInfosCache + federationPDUsCache + federationEDUsCache + spaceSummaryRoomsCache + lazyLoadingCache +) + +func NewRistrettoCache(maxCost config.DataUnit, maxAge time.Duration, enablePrometheus bool) *Caches { + cache, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e5, // 10x number of expected cache items, affects bloom filter size, gives us room for 10,000 currently + BufferItems: 64, // recommended by the ristretto godocs as a sane buffer size value + MaxCost: int64(maxCost), + Metrics: true, + KeyToHash: func(key interface{}) (uint64, uint64) { + return z.KeyToHash(key) + }, + }) + if err != nil { + panic(err) + } + if enablePrometheus { + promauto.NewGaugeFunc(prometheus.GaugeOpts{ + Namespace: "dendrite", + Subsystem: "caching_ristretto", + Name: "ratio", + }, func() float64 { + return float64(cache.Metrics.Ratio()) + }) + promauto.NewGaugeFunc(prometheus.GaugeOpts{ + Namespace: "dendrite", + Subsystem: "caching_ristretto", + Name: "cost", + }, func() float64 { + return float64(cache.Metrics.CostAdded() - cache.Metrics.CostEvicted()) + }) + } + return &Caches{ + RoomVersions: &RistrettoCachePartition[string, gomatrixserverlib.RoomVersion]{ // room ID -> room version + cache: cache, + Prefix: roomVersionsCache, + MaxAge: maxAge, + }, + ServerKeys: &RistrettoCachePartition[string, gomatrixserverlib.PublicKeyLookupResult]{ // server name -> server keys + cache: cache, + Prefix: serverKeysCache, + Mutable: true, + MaxAge: maxAge, + }, + RoomServerRoomNIDs: &RistrettoCachePartition[string, types.RoomNID]{ // room ID -> room NID + cache: cache, + Prefix: roomNIDsCache, + MaxAge: maxAge, + }, + RoomServerRoomIDs: &RistrettoCachePartition[int64, string]{ // room NID -> room ID + cache: cache, + Prefix: roomIDsCache, + MaxAge: maxAge, + }, + RoomServerEvents: &RistrettoCostedCachePartition[int64, *gomatrixserverlib.Event]{ // event NID -> event + &RistrettoCachePartition[int64, *gomatrixserverlib.Event]{ + cache: cache, + Prefix: roomEventsCache, + MaxAge: maxAge, + }, + }, + RoomInfos: &RistrettoCachePartition[string, types.RoomInfo]{ // room ID -> room info + cache: cache, + Prefix: roomInfosCache, + Mutable: true, + MaxAge: maxAge, + }, + FederationPDUs: &RistrettoCostedCachePartition[int64, *gomatrixserverlib.HeaderedEvent]{ // queue NID -> PDU + &RistrettoCachePartition[int64, *gomatrixserverlib.HeaderedEvent]{ + cache: cache, + Prefix: federationPDUsCache, + Mutable: true, + MaxAge: lesserOf(time.Hour/2, maxAge), + }, + }, + FederationEDUs: &RistrettoCostedCachePartition[int64, *gomatrixserverlib.EDU]{ // queue NID -> EDU + &RistrettoCachePartition[int64, *gomatrixserverlib.EDU]{ + cache: cache, + Prefix: federationEDUsCache, + Mutable: true, + MaxAge: lesserOf(time.Hour/2, maxAge), + }, + }, + SpaceSummaryRooms: &RistrettoCachePartition[string, gomatrixserverlib.MSC2946SpacesResponse]{ // room ID -> space response + cache: cache, + Prefix: spaceSummaryRoomsCache, + Mutable: true, + MaxAge: maxAge, + }, + LazyLoading: &RistrettoCachePartition[lazyLoadingCacheKey, string]{ // composite key -> event ID + cache: cache, + Prefix: lazyLoadingCache, + Mutable: true, + MaxAge: maxAge, + }, + } +} + +type RistrettoCostedCachePartition[k keyable, v costable] struct { + *RistrettoCachePartition[k, v] +} + +func (c *RistrettoCostedCachePartition[K, V]) Set(key K, value V) { + cost := value.CacheCost() + c.setWithCost(key, value, int64(cost)) +} + +type RistrettoCachePartition[K keyable, V any] struct { + cache *ristretto.Cache + Prefix byte + Mutable bool + MaxAge time.Duration +} + +func (c *RistrettoCachePartition[K, V]) setWithCost(key K, value V, cost int64) { + bkey := fmt.Sprintf("%c%v", c.Prefix, key) + if !c.Mutable { + if v, ok := c.cache.Get(bkey); ok && v != nil && !reflect.DeepEqual(v, value) { + panic(fmt.Sprintf("invalid use of immutable cache tries to change value of %v from %v to %v", key, v, value)) + } + } + c.cache.SetWithTTL(bkey, value, int64(len(bkey))+cost, c.MaxAge) +} + +func (c *RistrettoCachePartition[K, V]) Set(key K, value V) { + var cost int64 + if cv, ok := any(value).(string); ok { + cost = int64(len(cv)) + } else { + cost = int64(unsafe.Sizeof(value)) + } + c.setWithCost(key, value, cost) +} + +func (c *RistrettoCachePartition[K, V]) Unset(key K) { + bkey := fmt.Sprintf("%c%v", c.Prefix, key) + if !c.Mutable { + panic(fmt.Sprintf("invalid use of immutable cache tries to unset value of %v", key)) + } + c.cache.Del(bkey) +} + +func (c *RistrettoCachePartition[K, V]) Get(key K) (value V, ok bool) { + bkey := fmt.Sprintf("%c%v", c.Prefix, key) + v, ok := c.cache.Get(bkey) + if !ok || v == nil { + var empty V + return empty, false + } + value, ok = v.(V) + return +} + +func lesserOf(a, b time.Duration) time.Duration { + if a < b { + return a + } + return b +} diff --git a/internal/eventutil/events.go b/internal/eventutil/events.go index ee67a6daf..d96231963 100644 --- a/internal/eventutil/events.go +++ b/internal/eventutil/events.go @@ -170,20 +170,18 @@ func truncateAuthAndPrevEvents(auth, prev []gomatrixserverlib.EventReference) ( // RedactEvent redacts the given event and sets the unsigned field appropriately. This should be used by // downstream components to the roomserver when an OutputTypeRedactedEvent occurs. -func RedactEvent(redactionEvent, redactedEvent *gomatrixserverlib.Event) (*gomatrixserverlib.Event, error) { +func RedactEvent(redactionEvent, redactedEvent *gomatrixserverlib.Event) error { // sanity check if redactionEvent.Type() != gomatrixserverlib.MRoomRedaction { - return nil, fmt.Errorf("RedactEvent: redactionEvent isn't a redaction event, is '%s'", redactionEvent.Type()) + return fmt.Errorf("RedactEvent: redactionEvent isn't a redaction event, is '%s'", redactionEvent.Type()) } - r := redactedEvent.Redact() - err := r.SetUnsignedField("redacted_because", redactionEvent) - if err != nil { - return nil, err + redactedEvent.Redact() + if err := redactedEvent.SetUnsignedField("redacted_because", redactionEvent); err != nil { + return err } // NOTSPEC: sytest relies on this unspecced field existing :( - err = r.SetUnsignedField("redacted_by", redactionEvent.EventID()) - if err != nil { - return nil, err + if err := redactedEvent.SetUnsignedField("redacted_by", redactionEvent.EventID()); err != nil { + return err } - return r, nil + return nil } diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index 743b1efe6..866670d7a 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -319,11 +319,9 @@ func (r *Inputer) processRoomEvent( // if storing this event results in it being redacted then do so. if !isRejected && redactedEventID == event.EventID() { - r, rerr := eventutil.RedactEvent(redactionEvent, event) - if rerr != nil { + if err = eventutil.RedactEvent(redactionEvent, event); err != nil { return fmt.Errorf("eventutil.RedactEvent: %w", rerr) } - event = r } // For outliers we can stop after we've stored the event itself as it diff --git a/roomserver/internal/input/input_test.go b/roomserver/internal/input/input_test.go index 7c65f9eac..4708560ac 100644 --- a/roomserver/internal/input/input_test.go +++ b/roomserver/internal/input/input_test.go @@ -48,10 +48,6 @@ func TestSingleTransactionOnInput(t *testing.T) { Kind: api.KindOutlier, // don't panic if we generate an output event Event: event.Headered(gomatrixserverlib.RoomVersionV6), } - cache, err := caching.NewInMemoryLRUCache(false) - if err != nil { - t.Fatal(err) - } db, err := storage.Open( nil, &config.DatabaseOptions{ @@ -59,7 +55,7 @@ func TestSingleTransactionOnInput(t *testing.T) { MaxOpenConnections: 1, MaxIdleConnections: 1, }, - cache, + caching.NewRistrettoCache(8*1024*1024, time.Hour, false), ) if err != nil { t.Logf("PostgreSQL not available (%s), skipping", err) diff --git a/roomserver/internal/perform/perform_backfill.go b/roomserver/internal/perform/perform_backfill.go index 9eddca733..3f98fbc24 100644 --- a/roomserver/internal/perform/perform_backfill.go +++ b/roomserver/internal/perform/perform_backfill.go @@ -593,12 +593,11 @@ func persistEvents(ctx context.Context, db storage.Database, events []*gomatrixs // redacted, which we don't care about since we aren't returning it in this backfill. if redactedEventID == ev.EventID() { eventToRedact := ev.Unwrap() - redactedEvent, err := eventutil.RedactEvent(redactionEvent, eventToRedact) - if err != nil { + if err := eventutil.RedactEvent(redactionEvent, eventToRedact); err != nil { logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to redact event") continue } - ev = redactedEvent.Headered(ev.RoomVersion) + ev = eventToRedact.Headered(ev.RoomVersion) events[j] = ev } backfilledEventMap[ev.EventID()] = types.Event{ diff --git a/roomserver/state/state.go b/roomserver/state/state.go index 91f271652..d1d24b099 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -1027,7 +1027,7 @@ func (v *StateResolution) loadStateEvents( result := make([]*gomatrixserverlib.Event, 0, len(entries)) eventEntries := make([]types.StateEntry, 0, len(entries)) - eventNIDs := make([]types.EventNID, 0, len(entries)) + eventNIDs := make(types.EventNIDs, 0, len(entries)) for _, entry := range entries { if e, ok := v.events[entry.EventNID]; ok { result = append(result, e) diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index ba937ba30..692af1f6c 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -439,8 +439,18 @@ func (d *Database) Events( } func (d *Database) events( - ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID, + ctx context.Context, txn *sql.Tx, inputEventNIDs types.EventNIDs, ) ([]types.Event, error) { + sort.Sort(inputEventNIDs) + events := make(map[types.EventNID]*gomatrixserverlib.Event, len(inputEventNIDs)) + eventNIDs := make([]types.EventNID, 0, len(inputEventNIDs)) + for _, nid := range inputEventNIDs { + if event, ok := d.Cache.GetRoomServerEvent(nid); ok && event != nil { + events[nid] = event + } else { + eventNIDs = append(eventNIDs, nid) + } + } eventJSONs, err := d.EventJSONTable.BulkSelectEventJSON(ctx, txn, eventNIDs) if err != nil { return nil, err @@ -476,18 +486,29 @@ func (d *Database) events( for n, v := range dbRoomVersions { roomVersions[n] = v } - results := make([]types.Event, len(eventJSONs)) - for i, eventJSON := range eventJSONs { - result := &results[i] - result.EventNID = eventJSON.EventNID - roomNID := roomNIDs[result.EventNID] + for _, eventJSON := range eventJSONs { + roomNID := roomNIDs[eventJSON.EventNID] roomVersion := roomVersions[roomNID] - result.Event, err = gomatrixserverlib.NewEventFromTrustedJSONWithEventID( + events[eventJSON.EventNID], err = gomatrixserverlib.NewEventFromTrustedJSONWithEventID( eventIDs[eventJSON.EventNID], eventJSON.EventJSON, false, roomVersion, ) if err != nil { return nil, err } + if event := events[eventJSON.EventNID]; event != nil { + d.Cache.StoreRoomServerEvent(eventJSON.EventNID, event) + } + } + results := make([]types.Event, 0, len(inputEventNIDs)) + for _, nid := range inputEventNIDs { + event, ok := events[nid] + if !ok || event == nil { + return nil, fmt.Errorf("event %d missing", nid) + } + results = append(results, types.Event{ + EventNID: nid, + Event: event, + }) } if !redactionsArePermanent { d.applyRedactions(results) @@ -854,7 +875,7 @@ func (d *Database) handleRedactions( // mark the event as redacted if redactionsArePermanent { - redactedEvent.Event = redactedEvent.Redact() + redactedEvent.Redact() } err = redactedEvent.SetUnsignedField("redacted_because", redactionEvent) @@ -926,7 +947,7 @@ func (d *Database) loadRedactionPair( func (d *Database) applyRedactions(events []types.Event) { for i := range events { if result := gjson.GetBytes(events[i].Unsigned(), "redacted_because"); result.Exists() { - events[i].Event = events[i].Redact() + events[i].Redact() } } } diff --git a/setup/base/base.go b/setup/base/base.go index 5cbd7da9c..93ab87de1 100644 --- a/setup/base/base.go +++ b/setup/base/base.go @@ -161,11 +161,6 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base } } - cache, err := caching.NewInMemoryLRUCache(enableMetrics) - if err != nil { - logrus.WithError(err).Warnf("Failed to create cache") - } - var dnsCache *gomatrixserverlib.DNSCache if cfg.Global.DNSCache.Enabled { dnsCache = gomatrixserverlib.NewDNSCache( @@ -233,7 +228,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base UseHTTPAPIs: useHTTPAPIs, tracerCloser: closer, Cfg: cfg, - Caches: cache, + Caches: caching.NewRistrettoCache(cfg.Global.Cache.EstimatedMaxSize, cfg.Global.Cache.MaxAge, enableMetrics), DNSCache: dnsCache, PublicClientAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicClientPathPrefix).Subrouter().UseEncodedPath(), PublicFederationAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath(), diff --git a/setup/config/config_global.go b/setup/config/config_global.go index 9d4c1485e..ac1380a4e 100644 --- a/setup/config/config_global.go +++ b/setup/config/config_global.go @@ -2,6 +2,8 @@ package config import ( "math/rand" + "strconv" + "strings" "time" "github.com/matrix-org/gomatrixserverlib" @@ -73,6 +75,9 @@ type Global struct { // ReportStats configures opt-in anonymous stats reporting. ReportStats ReportStats `yaml:"report_stats"` + + // Configuration for the caches. + Cache Cache `yaml:"cache"` } func (c *Global) Defaults(generate bool) { @@ -90,6 +95,7 @@ func (c *Global) Defaults(generate bool) { c.Sentry.Defaults() c.ServerNotices.Defaults(generate) c.ReportStats.Defaults() + c.Cache.Defaults(generate) } func (c *Global) Verify(configErrs *ConfigErrors, isMonolith bool) { @@ -102,6 +108,7 @@ func (c *Global) Verify(configErrs *ConfigErrors, isMonolith bool) { c.DNSCache.Verify(configErrs, isMonolith) c.ServerNotices.Verify(configErrs, isMonolith) c.ReportStats.Verify(configErrs, isMonolith) + c.Cache.Verify(configErrs, isMonolith) } type OldVerifyKeys struct { @@ -168,6 +175,20 @@ func (c *ServerNotices) Defaults(generate bool) { func (c *ServerNotices) Verify(errors *ConfigErrors, isMonolith bool) {} +type Cache struct { + EstimatedMaxSize DataUnit `yaml:"max_size_estimated"` + MaxAge time.Duration `yaml:"max_age"` +} + +func (c *Cache) Defaults(generate bool) { + c.EstimatedMaxSize = 1024 * 1024 * 1024 // 1GB + c.MaxAge = time.Hour +} + +func (c *Cache) Verify(errors *ConfigErrors, isMonolith bool) { + checkPositive(errors, "max_size_estimated", int64(c.EstimatedMaxSize)) +} + // ReportStats configures opt-in anonymous stats reporting. type ReportStats struct { // Enabled configures anonymous usage stats of the server @@ -268,3 +289,28 @@ type PresenceOptions struct { // Whether outbound presence events are allowed EnableOutbound bool `yaml:"enable_outbound"` } + +type DataUnit int64 + +func (d *DataUnit) UnmarshalText(text []byte) error { + var magnitude float64 + s := strings.ToLower(string(text)) + switch { + case strings.HasSuffix(s, "tb"): + s, magnitude = s[:len(s)-2], 1024*1024*1024*1024 + case strings.HasSuffix(s, "gb"): + s, magnitude = s[:len(s)-2], 1024*1024*1024 + case strings.HasSuffix(s, "mb"): + s, magnitude = s[:len(s)-2], 1024*1024 + case strings.HasSuffix(s, "kb"): + s, magnitude = s[:len(s)-2], 1024 + default: + magnitude = 1 + } + v, err := strconv.ParseFloat(s, 64) + if err != nil { + return err + } + *d = DataUnit(v * magnitude) + return nil +} diff --git a/setup/config/config_test.go b/setup/config/config_test.go index cbc57ad18..b9b1e7bb5 100644 --- a/setup/config/config_test.go +++ b/setup/config/config_test.go @@ -17,6 +17,8 @@ package config import ( "fmt" "testing" + + "gopkg.in/yaml.v2" ) func TestLoadConfigRelative(t *testing.T) { @@ -268,3 +270,22 @@ n0Xq64k7fc42HXJpF8CGBkSaIhtlzcruO+vqR80B9r62+D0V7VmHOnP135MT6noU ANAf5kxmMsM0zlN2hkxl0H6o7wKlBSw3RI3cjfilXiMWRPJrzlc4 -----END CERTIFICATE----- ` + +func TestUnmarshalDataUnit(t *testing.T) { + target := struct { + Got DataUnit `yaml:"value"` + }{} + for input, expect := range map[string]DataUnit{ + "value: 0.6tb": 659706976665, + "value: 1.2gb": 1288490188, + "value: 256mb": 268435456, + "value: 128kb": 131072, + "value: 128": 128, + } { + if err := yaml.Unmarshal([]byte(input), &target); err != nil { + t.Fatal(err) + } else if target.Got != expect { + t.Fatalf("expected value %d but got %d", expect, target.Got) + } + } +} diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index ec5edd355..76114aff8 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -545,12 +545,11 @@ func (d *Database) RedactEvent(ctx context.Context, redactedEventID string, reda } eventToRedact := redactedEvents[0].Unwrap() redactionEvent := redactedBecause.Unwrap() - ev, err := eventutil.RedactEvent(redactionEvent, eventToRedact) - if err != nil { + if err = eventutil.RedactEvent(redactionEvent, eventToRedact); err != nil { return err } - newEvent := ev.Headered(redactedBecause.RoomVersion) + newEvent := eventToRedact.Headered(redactedBecause.RoomVersion) err = d.Writer.Do(nil, nil, func(txn *sql.Tx) error { return d.OutputEvents.UpdateEventJSON(ctx, newEvent) }) From 09f0ff14c88fec2ae5b3db79054378a8ced255a9 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Tue, 12 Jul 2022 08:23:58 +0200 Subject: [PATCH 04/19] Minor SendToDevice fix (#2565) * Avoid unnecessary marshalling if sending to the local server * Fix ordering of ToDevice messages * Revive SendToDevice test --- federationapi/consumers/sendtodevice.go | 5 + .../storage/postgres/send_to_device_table.go | 10 +- .../storage/sqlite3/send_to_device_table.go | 8 +- syncapi/storage/storage_test.go | 191 +++++++++++------- 4 files changed, 129 insertions(+), 85 deletions(-) diff --git a/federationapi/consumers/sendtodevice.go b/federationapi/consumers/sendtodevice.go index 84c9f620d..f99a895e0 100644 --- a/federationapi/consumers/sendtodevice.go +++ b/federationapi/consumers/sendtodevice.go @@ -95,6 +95,11 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msg *nats.Ms return true } + // The SyncAPI is already handling sendToDevice for the local server + if destServerName == t.ServerName { + return true + } + // Pack the EDU and marshal it edu := &gomatrixserverlib.EDU{ Type: gomatrixserverlib.MDirectToDevice, diff --git a/syncapi/storage/postgres/send_to_device_table.go b/syncapi/storage/postgres/send_to_device_table.go index 47c1cdaed..96d6844fd 100644 --- a/syncapi/storage/postgres/send_to_device_table.go +++ b/syncapi/storage/postgres/send_to_device_table.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" + "github.com/sirupsen/logrus" ) const sendToDeviceSchema = ` @@ -51,7 +52,7 @@ const selectSendToDeviceMessagesSQL = ` SELECT id, user_id, device_id, content FROM syncapi_send_to_device WHERE user_id = $1 AND device_id = $2 AND id > $3 AND id <= $4 - ORDER BY id DESC + ORDER BY id ASC ` const deleteSendToDeviceMessagesSQL = ` @@ -112,17 +113,18 @@ func (s *sendToDeviceStatements) SelectSendToDeviceMessages( if err = rows.Scan(&id, &userID, &deviceID, &content); err != nil { return } - if id > lastPos { - lastPos = id - } event := types.SendToDeviceEvent{ ID: id, UserID: userID, DeviceID: deviceID, } if err = json.Unmarshal([]byte(content), &event.SendToDeviceEvent); err != nil { + logrus.WithError(err).Errorf("Failed to unmarshal send-to-device message") continue } + if id > lastPos { + lastPos = id + } events = append(events, event) } if lastPos == 0 { diff --git a/syncapi/storage/sqlite3/send_to_device_table.go b/syncapi/storage/sqlite3/send_to_device_table.go index 0b1d5bbf2..5285acbe6 100644 --- a/syncapi/storage/sqlite3/send_to_device_table.go +++ b/syncapi/storage/sqlite3/send_to_device_table.go @@ -49,7 +49,7 @@ const selectSendToDeviceMessagesSQL = ` SELECT id, user_id, device_id, content FROM syncapi_send_to_device WHERE user_id = $1 AND device_id = $2 AND id > $3 AND id <= $4 - ORDER BY id DESC + ORDER BY id ASC ` const deleteSendToDeviceMessagesSQL = ` @@ -120,9 +120,6 @@ func (s *sendToDeviceStatements) SelectSendToDeviceMessages( logrus.WithError(err).Errorf("Failed to retrieve send-to-device message") return } - if id > lastPos { - lastPos = id - } event := types.SendToDeviceEvent{ ID: id, UserID: userID, @@ -132,6 +129,9 @@ func (s *sendToDeviceStatements) SelectSendToDeviceMessages( logrus.WithError(err).Errorf("Failed to unmarshal send-to-device message") continue } + if id > lastPos { + lastPos = id + } events = append(events, event) } if lastPos == 0 { diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index 563c92e34..c74151700 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -1,7 +1,9 @@ package storage_test import ( + "bytes" "context" + "encoding/json" "fmt" "reflect" "testing" @@ -394,90 +396,125 @@ func TestGetEventsInRangeWithEventsInsertedLikeBackfill(t *testing.T) { from = topologyTokenBefore(t, db, paginatedEvents[len(paginatedEvents)-1].EventID()) } } +*/ func TestSendToDeviceBehaviour(t *testing.T) { - //t.Parallel() - db := MustCreateDatabase(t) + t.Parallel() + alice := test.NewUser(t) + bob := test.NewUser(t) + deviceID := "one" + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, close := MustCreateDatabase(t, dbType) + defer close() + // At this point there should be no messages. We haven't sent anything + // yet. + _, events, err := db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, 100) + if err != nil { + t.Fatal(err) + } + if len(events) != 0 { + t.Fatal("first call should have no updates") + } - // At this point there should be no messages. We haven't sent anything - // yet. - _, events, updates, deletions, err := db.SendToDeviceUpdatesForSync(ctx, "alice", "one", types.StreamingToken{}) - if err != nil { - t.Fatal(err) - } - if len(events) != 0 || len(updates) != 0 || len(deletions) != 0 { - t.Fatal("first call should have no updates") - } - err = db.CleanSendToDeviceUpdates(context.Background(), updates, deletions, types.StreamingToken{}) - if err != nil { - return - } + err = db.CleanSendToDeviceUpdates(context.Background(), alice.ID, deviceID, 100) + if err != nil { + return + } - // Try sending a message. - streamPos, err := db.StoreNewSendForDeviceMessage(ctx, "alice", "one", gomatrixserverlib.SendToDeviceEvent{ - Sender: "bob", - Type: "m.type", - Content: json.RawMessage("{}"), + // Try sending a message. + streamPos, err := db.StoreNewSendForDeviceMessage(ctx, alice.ID, deviceID, gomatrixserverlib.SendToDeviceEvent{ + Sender: bob.ID, + Type: "m.type", + Content: json.RawMessage("{}"), + }) + if err != nil { + t.Fatal(err) + } + + // At this point we should get exactly one message. We're sending the sync position + // that we were given from the update and the send-to-device update will be updated + // in the database to reflect that this was the sync position we sent the message at. + streamPos, events, err = db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, streamPos) + if err != nil { + t.Fatal(err) + } + if count := len(events); count != 1 { + t.Fatalf("second call should have one update, got %d", count) + } + err = db.CleanSendToDeviceUpdates(context.Background(), alice.ID, deviceID, streamPos) + if err != nil { + return + } + + // At this point we should still have one message because we haven't progressed the + // sync position yet. This is equivalent to the client failing to /sync and retrying + // with the same position. + streamPos, events, err = db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, 100) + if err != nil { + t.Fatal(err) + } + if len(events) != 1 { + t.Fatal("third call should have one update still") + } + err = db.CleanSendToDeviceUpdates(context.Background(), alice.ID, deviceID, streamPos+1) + if err != nil { + return + } + + // At this point we should now have no updates, because we've progressed the sync + // position. Therefore the update from before will not be sent again. + _, events, err = db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, streamPos+1, streamPos+2) + if err != nil { + t.Fatal(err) + } + if len(events) != 0 { + t.Fatal("fourth call should have no updates") + } + err = db.CleanSendToDeviceUpdates(context.Background(), alice.ID, deviceID, streamPos+1) + if err != nil { + return + } + + // At this point we should still have no updates, because no new updates have been + // sent. + _, events, err = db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, streamPos, streamPos+2) + if err != nil { + t.Fatal(err) + } + if len(events) != 0 { + t.Fatal("fifth call should have no updates") + } + + // Send some more messages and verify the ordering is correct ("in order of arrival") + var lastPos types.StreamPosition = 0 + for i := 0; i < 10; i++ { + streamPos, err = db.StoreNewSendForDeviceMessage(ctx, alice.ID, deviceID, gomatrixserverlib.SendToDeviceEvent{ + Sender: bob.ID, + Type: "m.type", + Content: json.RawMessage(fmt.Sprintf(`{ "count": %d }`, i)), + }) + if err != nil { + t.Fatal(err) + } + lastPos = streamPos + } + + _, events, err = db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, lastPos) + if err != nil { + t.Fatalf("unable to get events: %v", err) + } + + for i := 0; i < 10; i++ { + want := json.RawMessage(fmt.Sprintf(`{"count":%d}`, i)) + got := events[i].Content + if !bytes.Equal(got, want) { + t.Fatalf("messages are out of order\nwant: %s\ngot: %s", string(want), string(got)) + } + } }) - if err != nil { - t.Fatal(err) - } - - // At this point we should get exactly one message. We're sending the sync position - // that we were given from the update and the send-to-device update will be updated - // in the database to reflect that this was the sync position we sent the message at. - _, events, updates, deletions, err = db.SendToDeviceUpdatesForSync(ctx, "alice", "one", types.StreamingToken{SendToDevicePosition: streamPos}) - if err != nil { - t.Fatal(err) - } - if len(events) != 1 || len(updates) != 1 || len(deletions) != 0 { - t.Fatal("second call should have one update") - } - err = db.CleanSendToDeviceUpdates(context.Background(), updates, deletions, types.StreamingToken{SendToDevicePosition: streamPos}) - if err != nil { - return - } - - // At this point we should still have one message because we haven't progressed the - // sync position yet. This is equivalent to the client failing to /sync and retrying - // with the same position. - _, events, updates, deletions, err = db.SendToDeviceUpdatesForSync(ctx, "alice", "one", types.StreamingToken{SendToDevicePosition: streamPos}) - if err != nil { - t.Fatal(err) - } - if len(events) != 1 || len(updates) != 0 || len(deletions) != 0 { - t.Fatal("third call should have one update still") - } - err = db.CleanSendToDeviceUpdates(context.Background(), updates, deletions, types.StreamingToken{SendToDevicePosition: streamPos}) - if err != nil { - return - } - - // At this point we should now have no updates, because we've progressed the sync - // position. Therefore the update from before will not be sent again. - _, events, updates, deletions, err = db.SendToDeviceUpdatesForSync(ctx, "alice", "one", types.StreamingToken{SendToDevicePosition: streamPos + 1}) - if err != nil { - t.Fatal(err) - } - if len(events) != 0 || len(updates) != 0 || len(deletions) != 1 { - t.Fatal("fourth call should have no updates") - } - err = db.CleanSendToDeviceUpdates(context.Background(), updates, deletions, types.StreamingToken{SendToDevicePosition: streamPos + 1}) - if err != nil { - return - } - - // At this point we should still have no updates, because no new updates have been - // sent. - _, events, updates, deletions, err = db.SendToDeviceUpdatesForSync(ctx, "alice", "one", types.StreamingToken{SendToDevicePosition: streamPos + 2}) - if err != nil { - t.Fatal(err) - } - if len(events) != 0 || len(updates) != 0 || len(deletions) != 0 { - t.Fatal("fifth call should have no updates") - } } +/* func TestInviteBehaviour(t *testing.T) { db := MustCreateDatabase(t) inviteRoom1 := "!inviteRoom1:somewhere" From 9cd8e9d4b97c6cff66665f57065a5bb8b7144045 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 13 Jul 2022 09:32:03 +0100 Subject: [PATCH 05/19] Use `/v3` to request media from remote servers (update to matrix-org/gomatrixserverlib#324) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2a2a037c1..1e62e1d93 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20220711125303-3bb2e997a44c + github.com/matrix-org/gomatrixserverlib v0.0.0-20220713083127-fc2ea1e62e46 github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.13 diff --git a/go.sum b/go.sum index 98549f702..86656ccd5 100644 --- a/go.sum +++ b/go.sum @@ -341,8 +341,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220711125303-3bb2e997a44c h1:mt30TDK8kXKV+nCmVfnqoXsh842N+74kvZw7DXuS/JQ= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220711125303-3bb2e997a44c/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220713083127-fc2ea1e62e46 h1:5X/kXY3nwqKOwwrE9tnMKrjbsi3PHigQYvrvDBSntO8= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220713083127-fc2ea1e62e46/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk= github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a h1:DdG8vXMlZ65EAtc4V+3t7zHZ2Gqs24pSnyXS+4BRHUs= github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From a1f9b02edfd56809eaedd04fd233b1828085e190 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 13 Jul 2022 10:13:34 +0100 Subject: [PATCH 06/19] Pointerise `types.RoomInfo` in the cache so we can update it in-place in the latest events updater --- internal/caching/cache_roominfo.go | 8 ++++---- internal/caching/caches.go | 2 +- internal/caching/impl_ristretto.go | 2 +- roomserver/storage/shared/room_updater.go | 13 ++++++------- roomserver/storage/shared/storage.go | 6 +++--- 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/internal/caching/cache_roominfo.go b/internal/caching/cache_roominfo.go index d03a61077..5dfed3c85 100644 --- a/internal/caching/cache_roominfo.go +++ b/internal/caching/cache_roominfo.go @@ -16,18 +16,18 @@ import ( // a room Info cache. It must only be used from the roomserver only // It is not safe for use from other components. type RoomInfoCache interface { - GetRoomInfo(roomID string) (roomInfo types.RoomInfo, ok bool) - StoreRoomInfo(roomID string, roomInfo types.RoomInfo) + GetRoomInfo(roomID string) (roomInfo *types.RoomInfo, ok bool) + StoreRoomInfo(roomID string, roomInfo *types.RoomInfo) } // GetRoomInfo must only be called from the roomserver only. It is not // safe for use from other components. -func (c Caches) GetRoomInfo(roomID string) (types.RoomInfo, bool) { +func (c Caches) GetRoomInfo(roomID string) (*types.RoomInfo, bool) { return c.RoomInfos.Get(roomID) } // StoreRoomInfo must only be called from the roomserver only. It is not // safe for use from other components. -func (c Caches) StoreRoomInfo(roomID string, roomInfo types.RoomInfo) { +func (c Caches) StoreRoomInfo(roomID string, roomInfo *types.RoomInfo) { c.RoomInfos.Set(roomID, roomInfo) } diff --git a/internal/caching/caches.go b/internal/caching/caches.go index 14b232dd0..e7914ce7d 100644 --- a/internal/caching/caches.go +++ b/internal/caching/caches.go @@ -28,7 +28,7 @@ type Caches struct { RoomServerRoomNIDs Cache[string, types.RoomNID] // room ID -> room NID RoomServerRoomIDs Cache[int64, string] // room NID -> room ID RoomServerEvents Cache[int64, *gomatrixserverlib.Event] // event NID -> event - RoomInfos Cache[string, types.RoomInfo] // room ID -> room info + RoomInfos Cache[string, *types.RoomInfo] // room ID -> room info FederationPDUs Cache[int64, *gomatrixserverlib.HeaderedEvent] // queue NID -> PDU FederationEDUs Cache[int64, *gomatrixserverlib.EDU] // queue NID -> EDU SpaceSummaryRooms Cache[string, gomatrixserverlib.MSC2946SpacesResponse] // room ID -> space response diff --git a/internal/caching/impl_ristretto.go b/internal/caching/impl_ristretto.go index 6d625b552..677218b5e 100644 --- a/internal/caching/impl_ristretto.go +++ b/internal/caching/impl_ristretto.go @@ -100,7 +100,7 @@ func NewRistrettoCache(maxCost config.DataUnit, maxAge time.Duration, enableProm MaxAge: maxAge, }, }, - RoomInfos: &RistrettoCachePartition[string, types.RoomInfo]{ // room ID -> room info + RoomInfos: &RistrettoCachePartition[string, *types.RoomInfo]{ // room ID -> room info cache: cache, Prefix: roomInfosCache, Mutable: true, diff --git a/roomserver/storage/shared/room_updater.go b/roomserver/storage/shared/room_updater.go index 8f4e011bf..c35ac653c 100644 --- a/roomserver/storage/shared/room_updater.go +++ b/roomserver/storage/shared/room_updater.go @@ -225,13 +225,12 @@ func (u *RoomUpdater) SetLatestEvents( if err := u.d.RoomsTable.UpdateLatestEventNIDs(u.ctx, txn, roomNID, eventNIDs, lastEventNIDSent, currentStateSnapshotNID); err != nil { return fmt.Errorf("u.d.RoomsTable.updateLatestEventNIDs: %w", err) } - if roomID, ok := u.d.Cache.GetRoomServerRoomID(roomNID); ok { - if roomInfo, ok := u.d.Cache.GetRoomInfo(roomID); ok { - roomInfo.StateSnapshotNID = currentStateSnapshotNID - roomInfo.IsStub = false - u.d.Cache.StoreRoomInfo(roomID, roomInfo) - } - } + + // Since it's entirely possible that this types.RoomInfo came from the + // cache, we should make sure to update that entry so that the next run + // works from live data. + u.roomInfo.StateSnapshotNID = currentStateSnapshotNID + u.roomInfo.IsStub = false return nil }) } diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index 692af1f6c..d8d5f67c8 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -139,13 +139,13 @@ func (d *Database) RoomInfo(ctx context.Context, roomID string) (*types.RoomInfo } func (d *Database) roomInfo(ctx context.Context, txn *sql.Tx, roomID string) (*types.RoomInfo, error) { - if roomInfo, ok := d.Cache.GetRoomInfo(roomID); ok { - return &roomInfo, nil + if roomInfo, ok := d.Cache.GetRoomInfo(roomID); ok && roomInfo != nil { + return roomInfo, nil } roomInfo, err := d.RoomsTable.SelectRoomInfo(ctx, txn, roomID) if err == nil && roomInfo != nil { d.Cache.StoreRoomServerRoomID(roomInfo.RoomNID, roomID) - d.Cache.StoreRoomInfo(roomID, *roomInfo) + d.Cache.StoreRoomInfo(roomID, roomInfo) } return roomInfo, err } From 950659320eba4d196032a6d63bcefcfbe1d8eaf9 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 14 Jul 2022 10:09:26 +0100 Subject: [PATCH 07/19] Add a Troubleshooting page --- docs/administration/5_troubleshooting.md | 81 ++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 docs/administration/5_troubleshooting.md diff --git a/docs/administration/5_troubleshooting.md b/docs/administration/5_troubleshooting.md new file mode 100644 index 000000000..14df2e3fb --- /dev/null +++ b/docs/administration/5_troubleshooting.md @@ -0,0 +1,81 @@ +--- +title: Troubleshooting +parent: Administration +permalink: /administration/troubleshooting +--- + +# Troubleshooting + +If your Dendrite installation is acting strangely, there are a few things you should +check before seeking help. + +## 1. Logs + +Dendrite, by default, will log all warnings and errors to stdout, in addition to any +other locations configured in the `dendrite.yaml` configuration file. Often there will +be clues in the logs. + +You can increase this log level to the more verbose `debug` level if necessary by adding +this to the config and restarting Dendrite: + +``` +logging: +- type: std + level: debug +``` + +Look specifically for lines that contain `level=error` or `level=warning`. + +## 2. Federation tester + +If you are experiencing problems federating with other homeservers, you should check +that the [Federation Tester](https://federationtester.matrix.org) is passing for your +server. + +Common reasons that it may not pass include: + +1. Incorrect DNS configuration; +2. Misconfigured DNS SRV entries or well-known files; +3. Invalid TLS/SSL certificates; +4. Reverse proxy configuration issues (if applicable). + +Correct any errors if shown and re-run the federation tester to check the results. + +## 3. System time + +Matrix relies heavily on TLS which requires the system time to be correct. If the clock +drifts then you may find that federation no works reliably (or at all) and clients may +struggle to connect to your Dendrite server. + +Ensure that your system time is correct and consider syncing to a reliable NTP source. + +## 4. Database connections + +If you are using the PostgreSQL database, you should ensure that Dendrite's configured +number of database connections does not exceed the maximum allowed by PostgreSQL. + +Open your `postgresql.conf` configuration file and check the value of `max_connections` +(which is typically `100` by default). Then open your `dendrite.yaml` configuration file +and ensure that: + +1. If you are using the `global.database` section, that `max_open_conns` does not exceed + that number; +2. If you are **not** using the `global.database` section, that the sum total of all + `max_open_conns` across all `database` blocks does not exceed that number. + +## 5. File descriptors + +Dendrite requires a sufficient number of file descriptors for every connection it makes +to a remote server, every connection to the database engine and every file it is reading +or writing to at a given time (media, logs etc). We recommend ensuring that the limit is +no lower than 65535 for Dendrite. + +Dendrite will check at startup if there are a sufficient number of available descriptors. +If there aren't, you will see a log lines like this: + +``` +level=warning msg="IMPORTANT: Process file descriptor limit is currently 65535, it is recommended to raise the limit for Dendrite to at least 65535 to avoid issues" +``` + +Follow the [Optimisation](../installation/10_optimisation.md) instructions to correct the +available number of file descriptors. From 69c86295f70b6756913df0bba37e30e0b3a6070e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 14 Jul 2022 13:51:00 +0100 Subject: [PATCH 08/19] Update `sytest-whitelist` --- sytest-whitelist | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sytest-whitelist b/sytest-whitelist index ea25c75d0..2a145291f 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -719,4 +719,5 @@ registration is idempotent, with username specified Setting state twice is idempotent Joining room twice is idempotent Inbound federation can return missing events for shared visibility -Inbound federation ignores redactions from invalid servers room > v3 \ No newline at end of file +Inbound federation ignores redactions from invalid servers room > v3 +Newly joined room includes presence in incremental sync \ No newline at end of file From 90bf01d8b107375d45aee9de79c1760a0363b189 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 15 Jul 2022 16:25:26 +0100 Subject: [PATCH 09/19] Use sync API database in `filterSharedUsers` (#2572) * Add function to the sync API storage package for filtering shared users * Use the database instead of asking the RS API * Fix unit tests * Fix map handling in `filterSharedUsers` --- syncapi/internal/keychange.go | 33 +++++++++--------- syncapi/internal/keychange_test.go | 31 +++++++++++++---- syncapi/storage/interface.go | 7 ++++ .../postgres/current_room_state_table.go | 30 ++++++++++++++++ syncapi/storage/shared/syncserver.go | 4 +++ .../sqlite3/current_room_state_table.go | 34 ++++++++++++++++++- syncapi/storage/tables/interface.go | 2 ++ syncapi/streams/stream_devicelist.go | 2 +- syncapi/sync/requestpool.go | 2 +- 9 files changed, 118 insertions(+), 27 deletions(-) diff --git a/syncapi/internal/keychange.go b/syncapi/internal/keychange.go index d96718d20..03df9285c 100644 --- a/syncapi/internal/keychange.go +++ b/syncapi/internal/keychange.go @@ -21,6 +21,7 @@ import ( keyapi "github.com/matrix-org/dendrite/keyserver/api" keytypes "github.com/matrix-org/dendrite/keyserver/types" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -46,7 +47,7 @@ func DeviceOTKCounts(ctx context.Context, keyAPI keyapi.SyncKeyAPI, userID, devi // was filled in, else false if there are no new device list changes because there is nothing to catch up on. The response MUST // be already filled in with join/leave information. func DeviceListCatchup( - ctx context.Context, keyAPI keyapi.SyncKeyAPI, rsAPI roomserverAPI.SyncRoomserverAPI, + ctx context.Context, db storage.SharedUsers, keyAPI keyapi.SyncKeyAPI, rsAPI roomserverAPI.SyncRoomserverAPI, userID string, res *types.Response, from, to types.StreamPosition, ) (newPos types.StreamPosition, hasNew bool, err error) { @@ -93,7 +94,7 @@ func DeviceListCatchup( queryRes.UserIDs = append(queryRes.UserIDs, leaveUserIDs...) queryRes.UserIDs = util.UniqueStrings(queryRes.UserIDs) var sharedUsersMap map[string]int - sharedUsersMap, queryRes.UserIDs = filterSharedUsers(ctx, rsAPI, userID, queryRes.UserIDs) + sharedUsersMap, queryRes.UserIDs = filterSharedUsers(ctx, db, userID, queryRes.UserIDs) util.GetLogger(ctx).Debugf( "QueryKeyChanges request off=%d,to=%d response off=%d uids=%v", offset, toOffset, queryRes.Offset, queryRes.UserIDs, @@ -215,30 +216,28 @@ func TrackChangedUsers( return changed, left, nil } +// filterSharedUsers takes a list of remote users whose keys have changed and filters +// it down to include only users who the requesting user shares a room with. func filterSharedUsers( - ctx context.Context, rsAPI roomserverAPI.SyncRoomserverAPI, userID string, usersWithChangedKeys []string, + ctx context.Context, db storage.SharedUsers, userID string, usersWithChangedKeys []string, ) (map[string]int, []string) { - var result []string - var sharedUsersRes roomserverAPI.QuerySharedUsersResponse - err := rsAPI.QuerySharedUsers(ctx, &roomserverAPI.QuerySharedUsersRequest{ - UserID: userID, - OtherUserIDs: usersWithChangedKeys, - }, &sharedUsersRes) + sharedUsersMap := make(map[string]int, len(usersWithChangedKeys)) + for _, userID := range usersWithChangedKeys { + sharedUsersMap[userID] = 0 + } + sharedUsers, err := db.SharedUsers(ctx, userID, usersWithChangedKeys) if err != nil { // default to all users so we do needless queries rather than miss some important device update return nil, usersWithChangedKeys } + for _, userID := range sharedUsers { + sharedUsersMap[userID]++ + } // We forcibly put ourselves in this list because we should be notified about our own device updates // and if we are in 0 rooms then we don't technically share any room with ourselves so we wouldn't // be notified about key changes. - sharedUsersRes.UserIDsToCount[userID] = 1 - - for _, uid := range usersWithChangedKeys { - if sharedUsersRes.UserIDsToCount[uid] > 0 { - result = append(result, uid) - } - } - return sharedUsersRes.UserIDsToCount, result + sharedUsersMap[userID] = 1 + return sharedUsersMap, sharedUsers } func joinedRooms(res *types.Response, userID string) []string { diff --git a/syncapi/internal/keychange_test.go b/syncapi/internal/keychange_test.go index 219b35e2c..79ed440e7 100644 --- a/syncapi/internal/keychange_test.go +++ b/syncapi/internal/keychange_test.go @@ -11,6 +11,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/types" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" ) var ( @@ -105,6 +106,22 @@ func (s *mockRoomserverAPI) QuerySharedUsers(ctx context.Context, req *api.Query return nil } +// This is actually a database function, but seeing as we track the state inside the +// *mockRoomserverAPI, we'll just comply with the interface here instead. +func (s *mockRoomserverAPI) SharedUsers(ctx context.Context, userID string, otherUserIDs []string) ([]string, error) { + commonUsers := []string{} + for _, members := range s.roomIDToJoinedMembers { + for _, member := range members { + for _, userID := range otherUserIDs { + if member == userID { + commonUsers = append(commonUsers, userID) + } + } + } + } + return util.UniqueStrings(commonUsers), nil +} + type wantCatchup struct { hasNew bool changed []string @@ -178,7 +195,7 @@ func TestKeyChangeCatchupOnJoinShareNewUser(t *testing.T) { "!another:room": {syncingUser}, }, } - _, hasNew, err := DeviceListCatchup(context.Background(), &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken) + _, hasNew, err := DeviceListCatchup(context.Background(), rsAPI, &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken) if err != nil { t.Fatalf("DeviceListCatchup returned an error: %s", err) } @@ -201,7 +218,7 @@ func TestKeyChangeCatchupOnLeaveShareLeftUser(t *testing.T) { "!another:room": {syncingUser}, }, } - _, hasNew, err := DeviceListCatchup(context.Background(), &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken) + _, hasNew, err := DeviceListCatchup(context.Background(), rsAPI, &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken) if err != nil { t.Fatalf("DeviceListCatchup returned an error: %s", err) } @@ -224,7 +241,7 @@ func TestKeyChangeCatchupOnJoinShareNoNewUsers(t *testing.T) { "!another:room": {syncingUser, existingUser}, }, } - _, hasNew, err := DeviceListCatchup(context.Background(), &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken) + _, hasNew, err := DeviceListCatchup(context.Background(), rsAPI, &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken) if err != nil { t.Fatalf("Catchup returned an error: %s", err) } @@ -246,7 +263,7 @@ func TestKeyChangeCatchupOnLeaveShareNoUsers(t *testing.T) { "!another:room": {syncingUser, existingUser}, }, } - _, hasNew, err := DeviceListCatchup(context.Background(), &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken) + _, hasNew, err := DeviceListCatchup(context.Background(), rsAPI, &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken) if err != nil { t.Fatalf("DeviceListCatchup returned an error: %s", err) } @@ -305,7 +322,7 @@ func TestKeyChangeCatchupNoNewJoinsButMessages(t *testing.T) { roomID: {syncingUser, existingUser}, }, } - _, hasNew, err := DeviceListCatchup(context.Background(), &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken) + _, hasNew, err := DeviceListCatchup(context.Background(), rsAPI, &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken) if err != nil { t.Fatalf("DeviceListCatchup returned an error: %s", err) } @@ -333,7 +350,7 @@ func TestKeyChangeCatchupChangeAndLeft(t *testing.T) { "!another:room": {syncingUser}, }, } - _, hasNew, err := DeviceListCatchup(context.Background(), &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken) + _, hasNew, err := DeviceListCatchup(context.Background(), rsAPI, &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken) if err != nil { t.Fatalf("Catchup returned an error: %s", err) } @@ -419,7 +436,7 @@ func TestKeyChangeCatchupChangeAndLeftSameRoom(t *testing.T) { }, } _, hasNew, err := DeviceListCatchup( - context.Background(), &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken, + context.Background(), rsAPI, &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken, ) if err != nil { t.Fatalf("DeviceListCatchup returned an error: %s", err) diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 5a036d889..055426030 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -27,6 +27,8 @@ import ( type Database interface { Presence + SharedUsers + MaxStreamPositionForPDUs(ctx context.Context) (types.StreamPosition, error) MaxStreamPositionForReceipts(ctx context.Context) (types.StreamPosition, error) MaxStreamPositionForInvites(ctx context.Context) (types.StreamPosition, error) @@ -165,3 +167,8 @@ type Presence interface { PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error) } + +type SharedUsers interface { + // SharedUsers returns a subset of otherUserIDs that share a room with userID. + SharedUsers(ctx context.Context, userID string, otherUserIDs []string) ([]string, error) +} diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index 8ee387b39..c4667bafd 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -107,6 +107,11 @@ const selectEventsWithEventIDsSQL = "" + "SELECT event_id, added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" + " FROM syncapi_current_room_state WHERE event_id = ANY($1)" +const selectSharedUsersSQL = "" + + "SELECT state_key FROM syncapi_current_room_state WHERE room_id = ANY(" + + " SELECT room_id FROM syncapi_current_room_state WHERE state_key = $1 AND membership='join'" + + ") AND state_key = ANY($2) AND membership='join';" + type currentRoomStateStatements struct { upsertRoomStateStmt *sql.Stmt deleteRoomStateByEventIDStmt *sql.Stmt @@ -118,6 +123,7 @@ type currentRoomStateStatements struct { selectJoinedUsersInRoomStmt *sql.Stmt selectEventsWithEventIDsStmt *sql.Stmt selectStateEventStmt *sql.Stmt + selectSharedUsersStmt *sql.Stmt } func NewPostgresCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, error) { @@ -156,6 +162,9 @@ func NewPostgresCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, erro if s.selectStateEventStmt, err = db.Prepare(selectStateEventSQL); err != nil { return nil, err } + if s.selectSharedUsersStmt, err = db.Prepare(selectSharedUsersSQL); err != nil { + return nil, err + } return s, nil } @@ -379,3 +388,24 @@ func (s *currentRoomStateStatements) SelectStateEvent( } return &ev, err } + +func (s *currentRoomStateStatements) SelectSharedUsers( + ctx context.Context, txn *sql.Tx, userID string, otherUserIDs []string, +) ([]string, error) { + stmt := sqlutil.TxStmt(txn, s.selectSharedUsersStmt) + rows, err := stmt.QueryContext(ctx, userID, otherUserIDs) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "selectSharedUsersStmt: rows.close() failed") + + var stateKey string + result := make([]string, 0, len(otherUserIDs)) + for rows.Next() { + if err := rows.Scan(&stateKey); err != nil { + return nil, err + } + result = append(result, stateKey) + } + return result, rows.Err() +} diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index 76114aff8..d1c5e2d16 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -176,6 +176,10 @@ func (d *Database) AllPeekingDevicesInRooms(ctx context.Context) (map[string][]t return d.Peeks.SelectPeekingDevices(ctx) } +func (d *Database) SharedUsers(ctx context.Context, userID string, otherUserIDs []string) ([]string, error) { + return d.CurrentRoomState.SelectSharedUsers(ctx, nil, userID, otherUserIDs) +} + func (d *Database) GetStateEvent( ctx context.Context, roomID, evType, stateKey string, ) (*gomatrixserverlib.HeaderedEvent, error) { diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index f0a1c7bb7..376c3a3d2 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -91,6 +91,11 @@ const selectEventsWithEventIDsSQL = "" + "SELECT event_id, added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" + " FROM syncapi_current_room_state WHERE event_id IN ($1)" +const selectSharedUsersSQL = "" + + "SELECT state_key FROM syncapi_current_room_state WHERE room_id = ANY(" + + " SELECT room_id FROM syncapi_current_room_state WHERE state_key = $1 AND membership='join'" + + ") AND state_key IN ($2) AND membership='join';" + type currentRoomStateStatements struct { db *sql.DB streamIDStatements *StreamIDStatements @@ -100,8 +105,9 @@ type currentRoomStateStatements struct { selectRoomIDsWithMembershipStmt *sql.Stmt selectRoomIDsWithAnyMembershipStmt *sql.Stmt selectJoinedUsersStmt *sql.Stmt - //selectJoinedUsersInRoomStmt *sql.Stmt - prepared at runtime due to variadic + //selectJoinedUsersInRoomStmt *sql.Stmt - prepared at runtime due to variadic selectStateEventStmt *sql.Stmt + //selectSharedUsersSQL *sql.Stmt - prepared at runtime due to variadic } func NewSqliteCurrentRoomStateTable(db *sql.DB, streamID *StreamIDStatements) (tables.CurrentRoomState, error) { @@ -396,3 +402,29 @@ func (s *currentRoomStateStatements) SelectStateEvent( } return &ev, err } + +func (s *currentRoomStateStatements) SelectSharedUsers( + ctx context.Context, txn *sql.Tx, userID string, otherUserIDs []string, +) ([]string, error) { + query := strings.Replace(selectSharedUsersSQL, "($2)", sqlutil.QueryVariadicOffset(len(otherUserIDs), 1), 1) + stmt, err := s.db.Prepare(query) + if err != nil { + return nil, fmt.Errorf("SelectSharedUsers s.db.Prepare: %w", err) + } + defer internal.CloseAndLogIfError(ctx, stmt, "SelectSharedUsers: stmt.close() failed") + rows, err := sqlutil.TxStmt(txn, stmt).QueryContext(ctx, userID, otherUserIDs) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "selectSharedUsersStmt: rows.close() failed") + + var stateKey string + result := make([]string, 0, len(otherUserIDs)) + for rows.Next() { + if err := rows.Scan(&stateKey); err != nil { + return nil, err + } + result = append(result, stateKey) + } + return result, rows.Err() +} diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index ccdebfdbd..08568d9ac 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -104,6 +104,8 @@ type CurrentRoomState interface { SelectJoinedUsers(ctx context.Context) (map[string][]string, error) // SelectJoinedUsersInRoom returns a map of room ID to a list of joined user IDs for a given room. SelectJoinedUsersInRoom(ctx context.Context, roomIDs []string) (map[string][]string, error) + // SelectSharedUsers returns a subset of otherUserIDs that share a room with userID. + SelectSharedUsers(ctx context.Context, txn *sql.Tx, userID string, otherUserIDs []string) ([]string, error) } // BackwardsExtremities keeps track of backwards extremities for a room. diff --git a/syncapi/streams/stream_devicelist.go b/syncapi/streams/stream_devicelist.go index f42099510..5448ee5bd 100644 --- a/syncapi/streams/stream_devicelist.go +++ b/syncapi/streams/stream_devicelist.go @@ -28,7 +28,7 @@ func (p *DeviceListStreamProvider) IncrementalSync( from, to types.StreamPosition, ) types.StreamPosition { var err error - to, _, err = internal.DeviceListCatchup(context.Background(), p.keyAPI, p.rsAPI, req.Device.UserID, req.Response, from, to) + to, _, err = internal.DeviceListCatchup(context.Background(), p.DB, p.keyAPI, p.rsAPI, req.Device.UserID, req.Response, from, to) if err != nil { req.Log.WithError(err).Error("internal.DeviceListCatchup failed") return from diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 6f0849e08..b6b4779a5 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -429,7 +429,7 @@ func (rp *RequestPool) OnIncomingKeyChangeRequest(req *http.Request, device *use } rp.streams.PDUStreamProvider.IncrementalSync(req.Context(), syncReq, fromToken.PDUPosition, toToken.PDUPosition) _, _, err = internal.DeviceListCatchup( - req.Context(), rp.keyAPI, rp.rsAPI, syncReq.Device.UserID, + req.Context(), rp.db, rp.keyAPI, rp.rsAPI, syncReq.Device.UserID, syncReq.Response, fromToken.DeviceListPosition, toToken.DeviceListPosition, ) if err != nil { From 4ff57993abc7066c52d979617a81af45629dd611 Mon Sep 17 00:00:00 2001 From: andreever <52261463+andreever@users.noreply.github.com> Date: Fri, 15 Jul 2022 17:33:05 +0200 Subject: [PATCH 10/19] Update 1_createusers.md (#2571) * Update 1_createusers.md Added description on how to create user accounts when running in docker. * Update 1_createusers.md Co-authored-by: Neil Alexander --- docs/administration/1_createusers.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/administration/1_createusers.md b/docs/administration/1_createusers.md index f40b7f576..61ec2299b 100644 --- a/docs/administration/1_createusers.md +++ b/docs/administration/1_createusers.md @@ -32,6 +32,15 @@ To create a new **admin account**, add the `-admin` flag: ./bin/create-account -config /path/to/dendrite.yaml -username USERNAME -admin ``` +An example of using `create-account` when running in **Docker**, having found the `CONTAINERNAME` from `docker ps`: + +```bash +docker exec -it CONTAINERNAME /usr/bin/create-account -config /path/to/dendrite.yaml -username USERNAME +``` +```bash +docker exec -it CONTAINERNAME /usr/bin/create-account -config /path/to/dendrite.yaml -username USERNAME -admin +``` + ## Using shared secret registration Dendrite supports the Synapse-compatible shared secret registration endpoint. From c0c909d30642d03600fd48bd052a22040a67267b Mon Sep 17 00:00:00 2001 From: Maximilian Gaedig <38767445+MaximilianGaedig@users.noreply.github.com> Date: Mon, 18 Jul 2022 00:01:52 +0200 Subject: [PATCH 11/19] Fix connection_string format in dendrite-sample.polylith.yaml (#2574) --- dendrite-sample.polylith.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dendrite-sample.polylith.yaml b/dendrite-sample.polylith.yaml index ea3d9d689..723547f55 100644 --- a/dendrite-sample.polylith.yaml +++ b/dendrite-sample.polylith.yaml @@ -125,7 +125,7 @@ app_service_api: # Database configuration for this component. database: - connection_string: postgresql://username@password:hostname/dendrite_appservice?sslmode=disable + connection_string: postgresql://username:password@hostname/dendrite_appservice?sslmode=disable max_open_conns: 10 max_idle_conns: 2 conn_max_lifetime: -1 @@ -199,7 +199,7 @@ federation_api: external_api: listen: http://[::]:8072 database: - connection_string: postgresql://username@password:hostname/dendrite_federationapi?sslmode=disable + connection_string: postgresql://username:password@hostname/dendrite_federationapi?sslmode=disable max_open_conns: 10 max_idle_conns: 2 conn_max_lifetime: -1 @@ -236,7 +236,7 @@ key_server: listen: http://[::]:7779 # The listen address for incoming API requests connect: http://key_server:7779 # The connect address for other components to use database: - connection_string: postgresql://username@password:hostname/dendrite_keyserver?sslmode=disable + connection_string: postgresql://username:password@hostname/dendrite_keyserver?sslmode=disable max_open_conns: 10 max_idle_conns: 2 conn_max_lifetime: -1 @@ -249,7 +249,7 @@ media_api: external_api: listen: http://[::]:8074 database: - connection_string: postgresql://username@password:hostname/dendrite_mediaapi?sslmode=disable + connection_string: postgresql://username:password@hostname/dendrite_mediaapi?sslmode=disable max_open_conns: 5 max_idle_conns: 2 conn_max_lifetime: -1 @@ -286,7 +286,7 @@ mscs: # - msc2836 # (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836) # - msc2946 # (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946) database: - connection_string: postgresql://username@password:hostname/dendrite_mscs?sslmode=disable + connection_string: postgresql://username:password@hostname/dendrite_mscs?sslmode=disable max_open_conns: 5 max_idle_conns: 2 conn_max_lifetime: -1 @@ -297,7 +297,7 @@ room_server: listen: http://[::]:7770 # The listen address for incoming API requests connect: http://room_server:7770 # The connect address for other components to use database: - connection_string: postgresql://username@password:hostname/dendrite_roomserver?sslmode=disable + connection_string: postgresql://username:password@hostname/dendrite_roomserver?sslmode=disable max_open_conns: 10 max_idle_conns: 2 conn_max_lifetime: -1 @@ -310,7 +310,7 @@ sync_api: external_api: listen: http://[::]:8073 database: - connection_string: postgresql://username@password:hostname/dendrite_syncapi?sslmode=disable + connection_string: postgresql://username:password@hostname/dendrite_syncapi?sslmode=disable max_open_conns: 10 max_idle_conns: 2 conn_max_lifetime: -1 @@ -326,7 +326,7 @@ user_api: listen: http://[::]:7781 # The listen address for incoming API requests connect: http://user_api:7781 # The connect address for other components to use account_database: - connection_string: postgresql://username@password:hostname/dendrite_userapi?sslmode=disable + connection_string: postgresql://username:password@hostname/dendrite_userapi?sslmode=disable max_open_conns: 10 max_idle_conns: 2 conn_max_lifetime: -1 From a7e92f8cb99105c2a579f73fbc972b7fb0b35678 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Mon, 18 Jul 2022 14:46:15 +0200 Subject: [PATCH 12/19] History visibility database changes (#2533) * Add new history_visibility column * Update SQL queries to include history_visibility * Store the history visibilty calculated by the roomserver * Update GMSL * Update migrations * Fix migration * Update GMSL * Fix `go.sum` * Update GMSL to use sql.Scanner & sql.Valuer * Re-order migration/table creation * Update gomatrixserverlib * Add history_visibility column to current_room_state * Fix migrations * Return error instead of Fatal log Co-authored-by: Neil Alexander --- go.mod | 2 +- go.sum | 4 +- syncapi/consumers/roomserver.go | 4 +- syncapi/routing/messages.go | 1 + syncapi/storage/interface.go | 4 +- .../postgres/current_room_state_table.go | 12 +-- ...2061412000000_history_visibility_column.go | 50 +++++++++++ .../postgres/output_room_events_table.go | 82 ++++++++++-------- syncapi/storage/postgres/syncserver.go | 24 ++++-- syncapi/storage/shared/syncserver.go | 7 +- .../sqlite3/current_room_state_table.go | 12 +-- ...2061412000000_history_visibility_column.go | 82 ++++++++++++++++++ .../sqlite3/output_room_events_table.go | 83 +++++++++++-------- syncapi/storage/sqlite3/syncserver.go | 24 ++++-- syncapi/storage/storage_test.go | 2 +- syncapi/storage/tables/interface.go | 9 +- .../storage/tables/output_room_events_test.go | 4 +- 17 files changed, 299 insertions(+), 107 deletions(-) create mode 100644 syncapi/storage/postgres/deltas/2022061412000000_history_visibility_column.go create mode 100644 syncapi/storage/sqlite3/deltas/2022061412000000_history_visibility_column.go diff --git a/go.mod b/go.mod index 1e62e1d93..0a965ec8e 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20220713083127-fc2ea1e62e46 + github.com/matrix-org/gomatrixserverlib v0.0.0-20220718085240-f08f98af7d2d github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.13 diff --git a/go.sum b/go.sum index 86656ccd5..df3feeb6e 100644 --- a/go.sum +++ b/go.sum @@ -341,8 +341,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220713083127-fc2ea1e62e46 h1:5X/kXY3nwqKOwwrE9tnMKrjbsi3PHigQYvrvDBSntO8= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220713083127-fc2ea1e62e46/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220718085240-f08f98af7d2d h1:BWInUURXVOW+OiifMapoRIS7i122KWdEKj6fnDFXgBo= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220718085240-f08f98af7d2d/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk= github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a h1:DdG8vXMlZ65EAtc4V+3t7zHZ2Gqs24pSnyXS+4BRHUs= github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index f0ca2106f..328e1e9dd 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -240,6 +240,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( msg.RemovesStateEventIDs, msg.TransactionID, false, + msg.HistoryVisibility, ) if err != nil { // panic rather than continue with an inconsistent database @@ -289,7 +290,8 @@ func (s *OutputRoomEventConsumer) onOldRoomEvent( []string{}, // adds no state []string{}, // removes no state nil, // no transaction - ev.StateKey() != nil, // exclude from sync? + ev.StateKey() != nil, // exclude from sync?, + msg.HistoryVisibility, ) if err != nil { // panic rather than continue with an inconsistent database diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 24745cd55..990ca55b1 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -594,6 +594,7 @@ func (r *messagesReq) backfill(roomID string, backwardsExtremities map[string][] []string{}, []string{}, nil, true, + gomatrixserverlib.HistoryVisibilityShared, ) if err != nil { return nil, err diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 055426030..9751670b2 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -69,7 +69,9 @@ type Database interface { // when generating the sync stream position for this event. Returns the sync stream position for the inserted event. // Returns an error if there was a problem inserting this event. WriteEvent(ctx context.Context, ev *gomatrixserverlib.HeaderedEvent, addStateEvents []*gomatrixserverlib.HeaderedEvent, - addStateEventIDs []string, removeStateEventIDs []string, transactionID *api.TransactionID, excludeFromSync bool) (types.StreamPosition, error) + addStateEventIDs []string, removeStateEventIDs []string, transactionID *api.TransactionID, excludeFromSync bool, + historyVisibility gomatrixserverlib.HistoryVisibility, + ) (types.StreamPosition, error) // PurgeRoomState completely purges room state from the sync API. This is done when // receiving an output event that completely resets the state. PurgeRoomState(ctx context.Context, roomID string) error diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index c4667bafd..aae2d8c33 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -51,6 +51,7 @@ CREATE TABLE IF NOT EXISTS syncapi_current_room_state ( -- The serial ID of the output_room_events table when this event became -- part of the current state of the room. added_at BIGINT, + history_visibility SMALLINT NOT NULL DEFAULT 2, -- Clobber based on 3-uple of room_id, type and state_key CONSTRAINT syncapi_room_state_unique UNIQUE (room_id, type, state_key) ); @@ -63,8 +64,8 @@ CREATE UNIQUE INDEX IF NOT EXISTS syncapi_current_room_state_eventid_idx ON sync ` const upsertRoomStateSQL = "" + - "INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, headered_event_json, membership, added_at)" + - " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" + + "INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, headered_event_json, membership, added_at, history_visibility)" + + " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)" + " ON CONFLICT ON CONSTRAINT syncapi_room_state_unique" + " DO UPDATE SET event_id = $2, sender=$4, contains_url=$5, headered_event_json = $7, membership = $8, added_at = $9" @@ -100,11 +101,11 @@ const selectStateEventSQL = "" + "SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3" const selectEventsWithEventIDsSQL = "" + - // TODO: The session_id and transaction_id blanks are here because otherwise - // the rowsToStreamEvents expects there to be exactly six columns. We need to + // TODO: The session_id and transaction_id blanks are here because + // the rowsToStreamEvents expects there to be exactly seven columns. We need to // figure out if these really need to be in the DB, and if so, we need a // better permanent fix for this. - neilalexander, 2 Jan 2020 - "SELECT event_id, added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" + + "SELECT event_id, added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id, history_visibility" + " FROM syncapi_current_room_state WHERE event_id = ANY($1)" const selectSharedUsersSQL = "" + @@ -336,6 +337,7 @@ func (s *currentRoomStateStatements) UpsertRoomState( headeredJSON, membership, addedAt, + event.Visibility, ) return err } diff --git a/syncapi/storage/postgres/deltas/2022061412000000_history_visibility_column.go b/syncapi/storage/postgres/deltas/2022061412000000_history_visibility_column.go new file mode 100644 index 000000000..2b4b65287 --- /dev/null +++ b/syncapi/storage/postgres/deltas/2022061412000000_history_visibility_column.go @@ -0,0 +1,50 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package deltas + +import ( + "database/sql" + "fmt" + + "github.com/matrix-org/dendrite/internal/sqlutil" +) + +func LoadAddHistoryVisibilityColumn(m *sqlutil.Migrations) { + m.AddMigration(UpAddHistoryVisibilityColumn, DownAddHistoryVisibilityColumn) +} + +func UpAddHistoryVisibilityColumn(tx *sql.Tx) error { + _, err := tx.Exec(` + ALTER TABLE syncapi_output_room_events ADD COLUMN IF NOT EXISTS history_visibility SMALLINT NOT NULL DEFAULT 2; + UPDATE syncapi_output_room_events SET history_visibility = 4 WHERE type IN ('m.room.message', 'm.room.encrypted'); + ALTER TABLE syncapi_current_room_state ADD COLUMN IF NOT EXISTS history_visibility SMALLINT NOT NULL DEFAULT 2; + UPDATE syncapi_current_room_state SET history_visibility = 4 WHERE type IN ('m.room.message', 'm.room.encrypted'); + `) + if err != nil { + return fmt.Errorf("failed to execute upgrade: %w", err) + } + return nil +} + +func DownAddHistoryVisibilityColumn(tx *sql.Tx) error { + _, err := tx.Exec(` + ALTER TABLE syncapi_output_room_events DROP COLUMN IF EXISTS history_visibility; + ALTER TABLE syncapi_current_room_state DROP COLUMN IF EXISTS history_visibility; + `) + if err != nil { + return fmt.Errorf("failed to execute downgrade: %w", err) + } + return nil +} diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index d84d0cfa2..d0474d4c3 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -67,7 +67,9 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( -- events retrieved through backfilling that have a position in the stream -- that relates to the moment these were retrieved rather than the moment these -- were emitted. - exclude_from_sync BOOL DEFAULT FALSE + exclude_from_sync BOOL DEFAULT FALSE, + -- The history visibility before this event (1 - world_readable; 2 - shared; 3 - invited; 4 - joined) + history_visibility SMALLINT NOT NULL DEFAULT 2 ); CREATE INDEX IF NOT EXISTS syncapi_output_room_events_type_idx ON syncapi_output_room_events (type); @@ -78,16 +80,16 @@ CREATE INDEX IF NOT EXISTS syncapi_output_room_events_exclude_from_sync_idx ON s const insertEventSQL = "" + "INSERT INTO syncapi_output_room_events (" + - "room_id, event_id, headered_event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id, exclude_from_sync" + - ") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) " + + "room_id, event_id, headered_event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id, exclude_from_sync, history_visibility" + + ") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) " + "ON CONFLICT ON CONSTRAINT syncapi_event_id_idx DO UPDATE SET exclude_from_sync = (excluded.exclude_from_sync AND $11) " + "RETURNING id" const selectEventsSQL = "" + - "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = ANY($1)" + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id, history_visibility FROM syncapi_output_room_events WHERE event_id = ANY($1)" const selectEventsWithFilterSQL = "" + - "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = ANY($1)" + + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id, history_visibility FROM syncapi_output_room_events WHERE event_id = ANY($1)" + " AND ( $2::text[] IS NULL OR sender = ANY($2) )" + " AND ( $3::text[] IS NULL OR NOT(sender = ANY($3)) )" + " AND ( $4::text[] IS NULL OR type LIKE ANY($4) )" + @@ -96,7 +98,7 @@ const selectEventsWithFilterSQL = "" + " LIMIT $7" const selectRecentEventsSQL = "" + - "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id, history_visibility FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + " AND ( $4::text[] IS NULL OR sender = ANY($4) )" + " AND ( $5::text[] IS NULL OR NOT(sender = ANY($5)) )" + @@ -105,7 +107,7 @@ const selectRecentEventsSQL = "" + " ORDER BY id DESC LIMIT $8" const selectRecentEventsForSyncSQL = "" + - "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id, history_visibility FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3 AND exclude_from_sync = FALSE" + " AND ( $4::text[] IS NULL OR sender = ANY($4) )" + " AND ( $5::text[] IS NULL OR NOT(sender = ANY($5)) )" + @@ -114,7 +116,7 @@ const selectRecentEventsForSyncSQL = "" + " ORDER BY id DESC LIMIT $8" const selectEarlyEventsSQL = "" + - "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id, history_visibility FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + " AND ( $4::text[] IS NULL OR sender = ANY($4) )" + " AND ( $5::text[] IS NULL OR NOT(sender = ANY($5)) )" + @@ -130,7 +132,7 @@ const updateEventJSONSQL = "" + // In order for us to apply the state updates correctly, rows need to be ordered in the order they were received (id). const selectStateInRangeSQL = "" + - "SELECT event_id, id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids" + + "SELECT event_id, id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids, history_visibility" + " FROM syncapi_output_room_events" + " WHERE (id > $1 AND id <= $2) AND (add_state_ids IS NOT NULL OR remove_state_ids IS NOT NULL)" + " AND room_id = ANY($3)" + @@ -146,10 +148,10 @@ const deleteEventsForRoomSQL = "" + "DELETE FROM syncapi_output_room_events WHERE room_id = $1" const selectContextEventSQL = "" + - "SELECT id, headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND event_id = $2" + "SELECT id, headered_event_json, history_visibility FROM syncapi_output_room_events WHERE room_id = $1 AND event_id = $2" const selectContextBeforeEventSQL = "" + - "SELECT headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND id < $2" + + "SELECT headered_event_json, history_visibility FROM syncapi_output_room_events WHERE room_id = $1 AND id < $2" + " AND ( $4::text[] IS NULL OR sender = ANY($4) )" + " AND ( $5::text[] IS NULL OR NOT(sender = ANY($5)) )" + " AND ( $6::text[] IS NULL OR type LIKE ANY($6) )" + @@ -157,7 +159,7 @@ const selectContextBeforeEventSQL = "" + " ORDER BY id DESC LIMIT $3" const selectContextAfterEventSQL = "" + - "SELECT id, headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND id > $2" + + "SELECT id, headered_event_json, history_visibility FROM syncapi_output_room_events WHERE room_id = $1 AND id > $2" + " AND ( $4::text[] IS NULL OR sender = ANY($4) )" + " AND ( $5::text[] IS NULL OR NOT(sender = ANY($5)) )" + " AND ( $6::text[] IS NULL OR type LIKE ANY($6) )" + @@ -246,14 +248,15 @@ func (s *outputRoomEventsStatements) SelectStateInRange( for rows.Next() { var ( - eventID string - streamPos types.StreamPosition - eventBytes []byte - excludeFromSync bool - addIDs pq.StringArray - delIDs pq.StringArray + eventID string + streamPos types.StreamPosition + eventBytes []byte + excludeFromSync bool + addIDs pq.StringArray + delIDs pq.StringArray + historyVisibility gomatrixserverlib.HistoryVisibility ) - if err := rows.Scan(&eventID, &streamPos, &eventBytes, &excludeFromSync, &addIDs, &delIDs); err != nil { + if err := rows.Scan(&eventID, &streamPos, &eventBytes, &excludeFromSync, &addIDs, &delIDs, &historyVisibility); err != nil { return nil, nil, err } // Sanity check for deleted state and whine if we see it. We don't need to do anything @@ -283,6 +286,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( needSet[id] = true } stateNeeded[ev.RoomID()] = needSet + ev.Visibility = historyVisibility eventIDToEvent[eventID] = types.StreamEvent{ HeaderedEvent: &ev, @@ -314,7 +318,7 @@ func (s *outputRoomEventsStatements) SelectMaxEventID( func (s *outputRoomEventsStatements) InsertEvent( ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, addState, removeState []string, - transactionID *api.TransactionID, excludeFromSync bool, + transactionID *api.TransactionID, excludeFromSync bool, historyVisibility gomatrixserverlib.HistoryVisibility, ) (streamPos types.StreamPosition, err error) { var txnID *string var sessionID *int64 @@ -351,6 +355,7 @@ func (s *outputRoomEventsStatements) InsertEvent( sessionID, txnID, excludeFromSync, + historyVisibility, ).Scan(&streamPos) return } @@ -504,13 +509,15 @@ func (s *outputRoomEventsStatements) SelectContextEvent(ctx context.Context, txn row := sqlutil.TxStmt(txn, s.selectContextEventStmt).QueryRowContext(ctx, roomID, eventID) var eventAsString string - if err = row.Scan(&id, &eventAsString); err != nil { + var historyVisibility gomatrixserverlib.HistoryVisibility + if err = row.Scan(&id, &eventAsString, &historyVisibility); err != nil { return 0, evt, err } if err = json.Unmarshal([]byte(eventAsString), &evt); err != nil { return 0, evt, err } + evt.Visibility = historyVisibility return id, evt, nil } @@ -532,15 +539,17 @@ func (s *outputRoomEventsStatements) SelectContextBeforeEvent( for rows.Next() { var ( - eventBytes []byte - evt *gomatrixserverlib.HeaderedEvent + eventBytes []byte + evt *gomatrixserverlib.HeaderedEvent + historyVisibility gomatrixserverlib.HistoryVisibility ) - if err = rows.Scan(&eventBytes); err != nil { + if err = rows.Scan(&eventBytes, &historyVisibility); err != nil { return evts, err } if err = json.Unmarshal(eventBytes, &evt); err != nil { return evts, err } + evt.Visibility = historyVisibility evts = append(evts, evt) } @@ -565,15 +574,17 @@ func (s *outputRoomEventsStatements) SelectContextAfterEvent( for rows.Next() { var ( - eventBytes []byte - evt *gomatrixserverlib.HeaderedEvent + eventBytes []byte + evt *gomatrixserverlib.HeaderedEvent + historyVisibility gomatrixserverlib.HistoryVisibility ) - if err = rows.Scan(&lastID, &eventBytes); err != nil { + if err = rows.Scan(&lastID, &eventBytes, &historyVisibility); err != nil { return 0, evts, err } if err = json.Unmarshal(eventBytes, &evt); err != nil { return 0, evts, err } + evt.Visibility = historyVisibility evts = append(evts, evt) } @@ -584,15 +595,16 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { var result []types.StreamEvent for rows.Next() { var ( - eventID string - streamPos types.StreamPosition - eventBytes []byte - excludeFromSync bool - sessionID *int64 - txnID *string - transactionID *api.TransactionID + eventID string + streamPos types.StreamPosition + eventBytes []byte + excludeFromSync bool + sessionID *int64 + txnID *string + transactionID *api.TransactionID + historyVisibility gomatrixserverlib.HistoryVisibility ) - if err := rows.Scan(&eventID, &streamPos, &eventBytes, &sessionID, &excludeFromSync, &txnID); err != nil { + if err := rows.Scan(&eventID, &streamPos, &eventBytes, &sessionID, &excludeFromSync, &txnID, &historyVisibility); err != nil { return nil, err } // TODO: Handle redacted events diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index 9cfe7c070..c6121e419 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -42,18 +42,16 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) if d.db, d.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter()); err != nil { return nil, err } + if _, err = d.db.Exec(outputRoomEventsSchema); err != nil { + return nil, err + } + if _, err = d.db.Exec(currentRoomStateSchema); err != nil { + return nil, err + } accountData, err := NewPostgresAccountDataTable(d.db) if err != nil { return nil, err } - events, err := NewPostgresEventsTable(d.db) - if err != nil { - return nil, err - } - currState, err := NewPostgresCurrentRoomStateTable(d.db) - if err != nil { - return nil, err - } invites, err := NewPostgresInvitesTable(d.db) if err != nil { return nil, err @@ -101,9 +99,19 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) m := sqlutil.NewMigrations() deltas.LoadFixSequences(m) deltas.LoadRemoveSendToDeviceSentColumn(m) + deltas.LoadAddHistoryVisibilityColumn(m) if err = m.RunDeltas(d.db, dbProperties); err != nil { return nil, err } + // prepare statements after the migrations have run + events, err := NewPostgresEventsTable(d.db) + if err != nil { + return nil, err + } + currState, err := NewPostgresCurrentRoomStateTable(d.db) + if err != nil { + return nil, err + } d.Database = shared.Database{ DB: d.db, Writer: d.writer, diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index d1c5e2d16..932bda164 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -368,11 +368,12 @@ func (d *Database) WriteEvent( addStateEvents []*gomatrixserverlib.HeaderedEvent, addStateEventIDs, removeStateEventIDs []string, transactionID *api.TransactionID, excludeFromSync bool, + historyVisibility gomatrixserverlib.HistoryVisibility, ) (pduPosition types.StreamPosition, returnErr error) { returnErr = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { var err error pos, err := d.OutputEvents.InsertEvent( - ctx, txn, ev, addStateEventIDs, removeStateEventIDs, transactionID, excludeFromSync, + ctx, txn, ev, addStateEventIDs, removeStateEventIDs, transactionID, excludeFromSync, historyVisibility, ) if err != nil { return fmt.Errorf("d.OutputEvents.InsertEvent: %w", err) @@ -391,7 +392,9 @@ func (d *Database) WriteEvent( // Nothing to do, the event may have just been a message event. return nil } - + for i := range addStateEvents { + addStateEvents[i].Visibility = historyVisibility + } return d.updateRoomState(ctx, txn, removeStateEventIDs, addStateEvents, pduPosition, topoPosition) }) diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index 376c3a3d2..501d0ee98 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -41,6 +41,7 @@ CREATE TABLE IF NOT EXISTS syncapi_current_room_state ( headered_event_json TEXT NOT NULL, membership TEXT, added_at BIGINT, + history_visibility SMALLINT NOT NULL DEFAULT 2, -- The history visibility before this event (1 - world_readable; 2 - shared; 3 - invited; 4 - joined) UNIQUE (room_id, type, state_key) ); -- for event deletion @@ -52,8 +53,8 @@ CREATE UNIQUE INDEX IF NOT EXISTS syncapi_current_room_state_eventid_idx ON sync ` const upsertRoomStateSQL = "" + - "INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, headered_event_json, membership, added_at)" + - " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" + + "INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, headered_event_json, membership, added_at, history_visibility)" + + " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)" + " ON CONFLICT (room_id, type, state_key)" + " DO UPDATE SET event_id = $2, sender=$4, contains_url=$5, headered_event_json = $7, membership = $8, added_at = $9" @@ -84,11 +85,11 @@ const selectStateEventSQL = "" + "SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3" const selectEventsWithEventIDsSQL = "" + - // TODO: The session_id and transaction_id blanks are here because otherwise - // the rowsToStreamEvents expects there to be exactly six columns. We need to + // TODO: The session_id and transaction_id blanks are here because + // the rowsToStreamEvents expects there to be exactly seven columns. We need to // figure out if these really need to be in the DB, and if so, we need a // better permanent fix for this. - neilalexander, 2 Jan 2020 - "SELECT event_id, added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" + + "SELECT event_id, added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id, history_visibility" + " FROM syncapi_current_room_state WHERE event_id IN ($1)" const selectSharedUsersSQL = "" + @@ -328,6 +329,7 @@ func (s *currentRoomStateStatements) UpsertRoomState( headeredJSON, membership, addedAt, + event.Visibility, ) return err } diff --git a/syncapi/storage/sqlite3/deltas/2022061412000000_history_visibility_column.go b/syncapi/storage/sqlite3/deltas/2022061412000000_history_visibility_column.go new file mode 100644 index 000000000..f6bcaddfb --- /dev/null +++ b/syncapi/storage/sqlite3/deltas/2022061412000000_history_visibility_column.go @@ -0,0 +1,82 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package deltas + +import ( + "database/sql" + "fmt" + + "github.com/matrix-org/dendrite/internal/sqlutil" +) + +func LoadAddHistoryVisibilityColumn(m *sqlutil.Migrations) { + m.AddMigration(UpAddHistoryVisibilityColumn, DownAddHistoryVisibilityColumn) +} + +func UpAddHistoryVisibilityColumn(tx *sql.Tx) error { + // SQLite doesn't have "if exists", so check if the column exists. If the query doesn't return an error, it already exists. + // Required for unit tests, as otherwise a duplicate column error will show up. + _, err := tx.Query("SELECT history_visibility FROM syncapi_output_room_events LIMIT 1") + if err == nil { + return nil + } + _, err = tx.Exec(` + ALTER TABLE syncapi_output_room_events ADD COLUMN history_visibility SMALLINT NOT NULL DEFAULT 2; + UPDATE syncapi_output_room_events SET history_visibility = 4 WHERE type IN ('m.room.message', 'm.room.encrypted'); + `) + if err != nil { + return fmt.Errorf("failed to execute upgrade: %w", err) + } + + _, err = tx.Query("SELECT history_visibility FROM syncapi_current_room_state LIMIT 1") + if err == nil { + return nil + } + _, err = tx.Exec(` + ALTER TABLE syncapi_current_room_state ADD COLUMN history_visibility SMALLINT NOT NULL DEFAULT 2; + UPDATE syncapi_current_room_state SET history_visibility = 4 WHERE type IN ('m.room.message', 'm.room.encrypted'); + `) + if err != nil { + return fmt.Errorf("failed to execute upgrade: %w", err) + } + return nil +} + +func DownAddHistoryVisibilityColumn(tx *sql.Tx) error { + // SQLite doesn't have "if exists", so check if the column exists. + _, err := tx.Query("SELECT history_visibility FROM syncapi_output_room_events LIMIT 1") + if err != nil { + // The column probably doesn't exist + return nil + } + _, err = tx.Exec(` + ALTER TABLE syncapi_output_room_events DROP COLUMN history_visibility; + `) + if err != nil { + return fmt.Errorf("failed to execute downgrade: %w", err) + } + _, err = tx.Query("SELECT history_visibility FROM syncapi_current_room_state LIMIT 1") + if err != nil { + // The column probably doesn't exist + return nil + } + _, err = tx.Exec(` + ALTER TABLE syncapi_current_room_state DROP COLUMN history_visibility; + `) + if err != nil { + return fmt.Errorf("failed to execute downgrade: %w", err) + } + return nil +} diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index f9961a9d1..b3dcb44cb 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -47,7 +47,8 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( remove_state_ids TEXT, -- JSON encoded string array session_id BIGINT, transaction_id TEXT, - exclude_from_sync BOOL NOT NULL DEFAULT FALSE + exclude_from_sync BOOL NOT NULL DEFAULT FALSE, + history_visibility SMALLINT NOT NULL DEFAULT 2 -- The history visibility before this event (1 - world_readable; 2 - shared; 3 - invited; 4 - joined) ); CREATE INDEX IF NOT EXISTS syncapi_output_room_events_type_idx ON syncapi_output_room_events (type); @@ -58,27 +59,27 @@ CREATE INDEX IF NOT EXISTS syncapi_output_room_events_exclude_from_sync_idx ON s const insertEventSQL = "" + "INSERT INTO syncapi_output_room_events (" + - "id, room_id, event_id, headered_event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id, exclude_from_sync" + - ") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) " + - "ON CONFLICT (event_id) DO UPDATE SET exclude_from_sync = (excluded.exclude_from_sync AND $13)" + "id, room_id, event_id, headered_event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id, exclude_from_sync, history_visibility" + + ") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) " + + "ON CONFLICT (event_id) DO UPDATE SET exclude_from_sync = (excluded.exclude_from_sync AND $14)" const selectEventsSQL = "" + - "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id IN ($1)" + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id, history_visibility FROM syncapi_output_room_events WHERE event_id IN ($1)" const selectRecentEventsSQL = "" + - "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id, history_visibility FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3" // WHEN, ORDER BY and LIMIT are appended by prepareWithFilters const selectRecentEventsForSyncSQL = "" + - "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id, history_visibility FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3 AND exclude_from_sync = FALSE" // WHEN, ORDER BY and LIMIT are appended by prepareWithFilters const selectEarlyEventsSQL = "" + - "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id, history_visibility FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3" // WHEN, ORDER BY and LIMIT are appended by prepareWithFilters @@ -90,7 +91,7 @@ const updateEventJSONSQL = "" + "UPDATE syncapi_output_room_events SET headered_event_json=$1 WHERE event_id=$2" const selectStateInRangeSQL = "" + - "SELECT event_id, id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids" + + "SELECT event_id, id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids, history_visibility" + " FROM syncapi_output_room_events" + " WHERE (id > $1 AND id <= $2)" + " AND room_id IN ($3)" + @@ -102,15 +103,15 @@ const deleteEventsForRoomSQL = "" + "DELETE FROM syncapi_output_room_events WHERE room_id = $1" const selectContextEventSQL = "" + - "SELECT id, headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND event_id = $2" + "SELECT id, headered_event_json, history_visibility FROM syncapi_output_room_events WHERE room_id = $1 AND event_id = $2" const selectContextBeforeEventSQL = "" + - "SELECT headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND id < $2" + "SELECT headered_event_json, history_visibility FROM syncapi_output_room_events WHERE room_id = $1 AND id < $2" // WHEN, ORDER BY and LIMIT are appended by prepareWithFilters const selectContextAfterEventSQL = "" + - "SELECT id, headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND id > $2" + "SELECT id, headered_event_json, history_visibility FROM syncapi_output_room_events WHERE room_id = $1 AND id > $2" // WHEN, ORDER BY and LIMIT are appended by prepareWithFilters @@ -196,14 +197,15 @@ func (s *outputRoomEventsStatements) SelectStateInRange( for rows.Next() { var ( - eventID string - streamPos types.StreamPosition - eventBytes []byte - excludeFromSync bool - addIDsJSON string - delIDsJSON string + eventID string + streamPos types.StreamPosition + eventBytes []byte + excludeFromSync bool + addIDsJSON string + delIDsJSON string + historyVisibility gomatrixserverlib.HistoryVisibility ) - if err := rows.Scan(&eventID, &streamPos, &eventBytes, &excludeFromSync, &addIDsJSON, &delIDsJSON); err != nil { + if err := rows.Scan(&eventID, &streamPos, &eventBytes, &excludeFromSync, &addIDsJSON, &delIDsJSON, &historyVisibility); err != nil { return nil, nil, err } @@ -239,6 +241,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( needSet[id] = true } stateNeeded[ev.RoomID()] = needSet + ev.Visibility = historyVisibility eventIDToEvent[eventID] = types.StreamEvent{ HeaderedEvent: &ev, @@ -270,7 +273,7 @@ func (s *outputRoomEventsStatements) SelectMaxEventID( func (s *outputRoomEventsStatements) InsertEvent( ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, addState, removeState []string, - transactionID *api.TransactionID, excludeFromSync bool, + transactionID *api.TransactionID, excludeFromSync bool, historyVisibility gomatrixserverlib.HistoryVisibility, ) (types.StreamPosition, error) { var txnID *string var sessionID *int64 @@ -326,6 +329,7 @@ func (s *outputRoomEventsStatements) InsertEvent( sessionID, txnID, excludeFromSync, + historyVisibility, excludeFromSync, ) return streamPos, err @@ -481,15 +485,16 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { var result []types.StreamEvent for rows.Next() { var ( - eventID string - streamPos types.StreamPosition - eventBytes []byte - excludeFromSync bool - sessionID *int64 - txnID *string - transactionID *api.TransactionID + eventID string + streamPos types.StreamPosition + eventBytes []byte + excludeFromSync bool + sessionID *int64 + txnID *string + transactionID *api.TransactionID + historyVisibility gomatrixserverlib.HistoryVisibility ) - if err := rows.Scan(&eventID, &streamPos, &eventBytes, &sessionID, &excludeFromSync, &txnID); err != nil { + if err := rows.Scan(&eventID, &streamPos, &eventBytes, &sessionID, &excludeFromSync, &txnID, &historyVisibility); err != nil { return nil, err } // TODO: Handle redacted events @@ -505,6 +510,8 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { } } + ev.Visibility = historyVisibility + result = append(result, types.StreamEvent{ HeaderedEvent: &ev, StreamPosition: streamPos, @@ -519,13 +526,15 @@ func (s *outputRoomEventsStatements) SelectContextEvent( ) (id int, evt gomatrixserverlib.HeaderedEvent, err error) { row := sqlutil.TxStmt(txn, s.selectContextEventStmt).QueryRowContext(ctx, roomID, eventID) var eventAsString string - if err = row.Scan(&id, &eventAsString); err != nil { + var historyVisibility gomatrixserverlib.HistoryVisibility + if err = row.Scan(&id, &eventAsString, &historyVisibility); err != nil { return 0, evt, err } if err = json.Unmarshal([]byte(eventAsString), &evt); err != nil { return 0, evt, err } + evt.Visibility = historyVisibility return id, evt, nil } @@ -550,15 +559,17 @@ func (s *outputRoomEventsStatements) SelectContextBeforeEvent( for rows.Next() { var ( - eventBytes []byte - evt *gomatrixserverlib.HeaderedEvent + eventBytes []byte + evt *gomatrixserverlib.HeaderedEvent + historyVisibility gomatrixserverlib.HistoryVisibility ) - if err = rows.Scan(&eventBytes); err != nil { + if err = rows.Scan(&eventBytes, &historyVisibility); err != nil { return evts, err } if err = json.Unmarshal(eventBytes, &evt); err != nil { return evts, err } + evt.Visibility = historyVisibility evts = append(evts, evt) } @@ -586,15 +597,17 @@ func (s *outputRoomEventsStatements) SelectContextAfterEvent( for rows.Next() { var ( - eventBytes []byte - evt *gomatrixserverlib.HeaderedEvent + eventBytes []byte + evt *gomatrixserverlib.HeaderedEvent + historyVisibility gomatrixserverlib.HistoryVisibility ) - if err = rows.Scan(&lastID, &eventBytes); err != nil { + if err = rows.Scan(&lastID, &eventBytes, &historyVisibility); err != nil { return 0, evts, err } if err = json.Unmarshal(eventBytes, &evt); err != nil { return 0, evts, err } + evt.Visibility = historyVisibility evts = append(evts, evt) } return lastID, evts, rows.Err() diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index e08a0ba82..39ceec81d 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -52,18 +52,16 @@ func (d *SyncServerDatasource) prepare(dbProperties *config.DatabaseOptions) (er if err = d.streamID.Prepare(d.db); err != nil { return err } + if _, err = d.db.Exec(outputRoomEventsSchema); err != nil { + return err + } + if _, err = d.db.Exec(currentRoomStateSchema); err != nil { + return err + } accountData, err := NewSqliteAccountDataTable(d.db, &d.streamID) if err != nil { return err } - events, err := NewSqliteEventsTable(d.db, &d.streamID) - if err != nil { - return err - } - roomState, err := NewSqliteCurrentRoomStateTable(d.db, &d.streamID) - if err != nil { - return err - } invites, err := NewSqliteInvitesTable(d.db, &d.streamID) if err != nil { return err @@ -111,9 +109,19 @@ func (d *SyncServerDatasource) prepare(dbProperties *config.DatabaseOptions) (er m := sqlutil.NewMigrations() deltas.LoadFixSequences(m) deltas.LoadRemoveSendToDeviceSentColumn(m) + deltas.LoadAddHistoryVisibilityColumn(m) if err = m.RunDeltas(d.db, dbProperties); err != nil { return err } + // prepare statements after the migrations have run + events, err := NewSqliteEventsTable(d.db, &d.streamID) + if err != nil { + return err + } + roomState, err := NewSqliteCurrentRoomStateTable(d.db, &d.streamID) + if err != nil { + return err + } d.Database = shared.Database{ DB: d.db, Writer: d.writer, diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index c74151700..df03a33c2 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -37,7 +37,7 @@ func MustWriteEvents(t *testing.T, db storage.Database, events []*gomatrixserver addStateEvents = append(addStateEvents, ev) addStateEventIDs = append(addStateEventIDs, ev.EventID()) } - pos, err := db.WriteEvent(ctx, ev, addStateEvents, addStateEventIDs, removeStateEventIDs, nil, false) + pos, err := db.WriteEvent(ctx, ev, addStateEvents, addStateEventIDs, removeStateEventIDs, nil, false, gomatrixserverlib.HistoryVisibilityShared) if err != nil { t.Fatalf("WriteEvent failed: %s", err) } diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index 08568d9ac..d8fc8f508 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -52,7 +52,14 @@ type Peeks interface { type Events interface { SelectStateInRange(ctx context.Context, txn *sql.Tx, r types.Range, stateFilter *gomatrixserverlib.StateFilter, roomIDs []string) (map[string]map[string]bool, map[string]types.StreamEvent, error) SelectMaxEventID(ctx context.Context, txn *sql.Tx) (id int64, err error) - InsertEvent(ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, addState, removeState []string, transactionID *api.TransactionID, excludeFromSync bool) (streamPos types.StreamPosition, err error) + InsertEvent( + ctx context.Context, txn *sql.Tx, + event *gomatrixserverlib.HeaderedEvent, + addState, removeState []string, + transactionID *api.TransactionID, + excludeFromSync bool, + historyVisibility gomatrixserverlib.HistoryVisibility, + ) (streamPos types.StreamPosition, err error) // SelectRecentEvents returns events between the two stream positions: exclusive of low and inclusive of high. // If onlySyncEvents has a value of true, only returns the events that aren't marked as to exclude from sync. // Returns up to `limit` events. Returns `limited=true` if there are more events in this range but we hit the `limit`. diff --git a/syncapi/storage/tables/output_room_events_test.go b/syncapi/storage/tables/output_room_events_test.go index 69bbd04c9..bdb17ae20 100644 --- a/syncapi/storage/tables/output_room_events_test.go +++ b/syncapi/storage/tables/output_room_events_test.go @@ -53,7 +53,7 @@ func TestOutputRoomEventsTable(t *testing.T) { events := room.Events() err := sqlutil.WithTransaction(db, func(txn *sql.Tx) error { for _, ev := range events { - _, err := tab.InsertEvent(ctx, txn, ev, nil, nil, nil, false) + _, err := tab.InsertEvent(ctx, txn, ev, nil, nil, nil, false, gomatrixserverlib.HistoryVisibilityShared) if err != nil { return fmt.Errorf("failed to InsertEvent: %s", err) } @@ -79,7 +79,7 @@ func TestOutputRoomEventsTable(t *testing.T) { "body": "test.txt", "url": "mxc://test.txt", }) - if _, err = tab.InsertEvent(ctx, txn, urlEv, nil, nil, nil, false); err != nil { + if _, err = tab.InsertEvent(ctx, txn, urlEv, nil, nil, nil, false, gomatrixserverlib.HistoryVisibilityShared); err != nil { return fmt.Errorf("failed to InsertEvent: %s", err) } wantEventID := []string{urlEv.EventID()} From 99b696e775f13eb876f68195acdbffc2e05764b2 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 18 Jul 2022 14:49:41 +0100 Subject: [PATCH 13/19] Tweak cache counters (#2575) * Tweak cache counters This makes the number of counters relative to the maximum cache size. Since the counters effectively manage the size of the bloom filter, larger caches need more counters and smaller caches need less. 10 counters per 1KB data means that the default cache size of 1GB should result in a bloom filter and TinyLRU admission set of about 16MB estimated. * Remove line left by accident --- internal/caching/impl_ristretto.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/caching/impl_ristretto.go b/internal/caching/impl_ristretto.go index 677218b5e..01c97ad55 100644 --- a/internal/caching/impl_ristretto.go +++ b/internal/caching/impl_ristretto.go @@ -44,9 +44,9 @@ const ( func NewRistrettoCache(maxCost config.DataUnit, maxAge time.Duration, enablePrometheus bool) *Caches { cache, err := ristretto.NewCache(&ristretto.Config{ - NumCounters: 1e5, // 10x number of expected cache items, affects bloom filter size, gives us room for 10,000 currently - BufferItems: 64, // recommended by the ristretto godocs as a sane buffer size value - MaxCost: int64(maxCost), + NumCounters: int64((maxCost / 1024) * 10), // 10 counters per 1KB data, affects bloom filter size + BufferItems: 64, // recommended by the ristretto godocs as a sane buffer size value + MaxCost: int64(maxCost), // max cost is in bytes, as per the Dendrite config Metrics: true, KeyToHash: func(key interface{}) (uint64, uint64) { return z.KeyToHash(key) From bcff14adea471cbb496924622295c62c02a82720 Mon Sep 17 00:00:00 2001 From: Till Faelligen Date: Mon, 18 Jul 2022 18:19:44 +0200 Subject: [PATCH 14/19] Set historyVisibility in rowsToStreamEvents --- syncapi/storage/postgres/output_room_events_table.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index d0474d4c3..ddef27383 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -619,7 +619,7 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { TransactionID: *txnID, } } - + ev.Visibility = historyVisibility result = append(result, types.StreamEvent{ HeaderedEvent: &ev, StreamPosition: streamPos, From 583b8ea273be82ebdf2a9f81fd51a150d910b349 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 19 Jul 2022 11:51:46 +0100 Subject: [PATCH 15/19] Update FAQ --- docs/FAQ.md | 9 ++++++--- setup/config/config_global.go | 6 +++--- userapi/util/phonehomestats.go | 6 +++--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/docs/FAQ.md b/docs/FAQ.md index 47f39b9e6..f8255684e 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -86,9 +86,12 @@ would be a huge help too, as that will help us to understand where the memory us You may need to revisit the connection limit of your PostgreSQL server and/or make changes to the `max_connections` lines in your Dendrite configuration. Be aware that each Dendrite component opens its own database connections and has its own connection limit, even in monolith mode! -## What is being reported when enabling anonymous stats? +## What is being reported when enabling phone-home statistics? -If anonymous stats reporting is enabled, the following data is send to the defined endpoint. +Phone-home statistics contain your server's domain name, some configuration information about +your deployment and aggregated information about active users on your deployment. They are sent +to the endpoint URL configured in your Dendrite configuration file only. The following is an +example of the data that is sent: ```json { @@ -106,7 +109,7 @@ If anonymous stats reporting is enabled, the following data is send to the defin "go_arch": "amd64", "go_os": "linux", "go_version": "go1.16.13", - "homeserver": "localhost:8800", + "homeserver": "my.domain.com", "log_level": "trace", "memory_rss": 93452, "monolith": true, diff --git a/setup/config/config_global.go b/setup/config/config_global.go index ac1380a4e..10827bb21 100644 --- a/setup/config/config_global.go +++ b/setup/config/config_global.go @@ -73,7 +73,7 @@ type Global struct { // ServerNotices configuration used for sending server notices ServerNotices ServerNotices `yaml:"server_notices"` - // ReportStats configures opt-in anonymous stats reporting. + // ReportStats configures opt-in phone-home statistics reporting. ReportStats ReportStats `yaml:"report_stats"` // Configuration for the caches. @@ -189,9 +189,9 @@ func (c *Cache) Verify(errors *ConfigErrors, isMonolith bool) { checkPositive(errors, "max_size_estimated", int64(c.EstimatedMaxSize)) } -// ReportStats configures opt-in anonymous stats reporting. +// ReportStats configures opt-in phone-home statistics reporting. type ReportStats struct { - // Enabled configures anonymous usage stats of the server + // Enabled configures phone-home statistics of the server Enabled bool `yaml:"enabled"` // Endpoint the endpoint to report stats to diff --git a/userapi/util/phonehomestats.go b/userapi/util/phonehomestats.go index ad93a50e3..b17f62060 100644 --- a/userapi/util/phonehomestats.go +++ b/userapi/util/phonehomestats.go @@ -139,7 +139,7 @@ func (p *phoneHomeStats) collect() { output := bytes.Buffer{} if err = json.NewEncoder(&output).Encode(p.stats); err != nil { - logrus.WithError(err).Error("unable to encode anonymous stats") + logrus.WithError(err).Error("Unable to encode phone-home statistics") return } @@ -147,14 +147,14 @@ func (p *phoneHomeStats) collect() { request, err := http.NewRequestWithContext(ctx, http.MethodPost, p.cfg.Global.ReportStats.Endpoint, &output) if err != nil { - logrus.WithError(err).Error("unable to create anonymous stats request") + logrus.WithError(err).Error("Unable to create phone-home statistics request") return } request.Header.Set("User-Agent", "Dendrite/"+internal.VersionString()) _, err = p.client.Do(request) if err != nil { - logrus.WithError(err).Error("unable to send anonymous stats") + logrus.WithError(err).Error("Unable to send phone-home statistics") return } } From 5c01306bb5add6f53907950982fc468c00ae266d Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 19 Jul 2022 12:15:48 +0100 Subject: [PATCH 16/19] Add event state key cache (#2576) --- internal/caching/cache_eventstatekeys.go | 18 ++++++++++++++++++ internal/caching/cache_roomservernids.go | 5 +++-- internal/caching/caches.go | 23 ++++++++++++----------- internal/caching/impl_ristretto.go | 8 +++++++- roomserver/storage/shared/storage.go | 19 ++++++++++++++++++- 5 files changed, 58 insertions(+), 15 deletions(-) create mode 100644 internal/caching/cache_eventstatekeys.go diff --git a/internal/caching/cache_eventstatekeys.go b/internal/caching/cache_eventstatekeys.go new file mode 100644 index 000000000..05580ab05 --- /dev/null +++ b/internal/caching/cache_eventstatekeys.go @@ -0,0 +1,18 @@ +package caching + +import "github.com/matrix-org/dendrite/roomserver/types" + +// EventStateKeyCache contains the subset of functions needed for +// a room event state key cache. +type EventStateKeyCache interface { + GetEventStateKey(eventStateKeyNID types.EventStateKeyNID) (string, bool) + StoreEventStateKey(eventStateKeyNID types.EventStateKeyNID, eventStateKey string) +} + +func (c Caches) GetEventStateKey(eventStateKeyNID types.EventStateKeyNID) (string, bool) { + return c.RoomServerStateKeys.Get(eventStateKeyNID) +} + +func (c Caches) StoreEventStateKey(eventStateKeyNID types.EventStateKeyNID, eventStateKey string) { + c.RoomServerStateKeys.Set(eventStateKeyNID, eventStateKey) +} diff --git a/internal/caching/cache_roomservernids.go b/internal/caching/cache_roomservernids.go index b409aeef2..f27154f19 100644 --- a/internal/caching/cache_roomservernids.go +++ b/internal/caching/cache_roomservernids.go @@ -9,6 +9,7 @@ type RoomServerCaches interface { RoomVersionCache RoomInfoCache RoomServerEventsCache + EventStateKeyCache } // RoomServerNIDsCache contains the subset of functions needed for @@ -19,9 +20,9 @@ type RoomServerNIDsCache interface { } func (c Caches) GetRoomServerRoomID(roomNID types.RoomNID) (string, bool) { - return c.RoomServerRoomIDs.Get(int64(roomNID)) + return c.RoomServerRoomIDs.Get(roomNID) } func (c Caches) StoreRoomServerRoomID(roomNID types.RoomNID, roomID string) { - c.RoomServerRoomIDs.Set(int64(roomNID), roomID) + c.RoomServerRoomIDs.Set(roomNID, roomID) } diff --git a/internal/caching/caches.go b/internal/caching/caches.go index e7914ce7d..f13f743d3 100644 --- a/internal/caching/caches.go +++ b/internal/caching/caches.go @@ -23,16 +23,17 @@ import ( // different implementations as long as they satisfy the Cache // interface. type Caches struct { - RoomVersions Cache[string, gomatrixserverlib.RoomVersion] // room ID -> room version - ServerKeys Cache[string, gomatrixserverlib.PublicKeyLookupResult] // server name -> server keys - RoomServerRoomNIDs Cache[string, types.RoomNID] // room ID -> room NID - RoomServerRoomIDs Cache[int64, string] // room NID -> room ID - RoomServerEvents Cache[int64, *gomatrixserverlib.Event] // event NID -> event - RoomInfos Cache[string, *types.RoomInfo] // room ID -> room info - FederationPDUs Cache[int64, *gomatrixserverlib.HeaderedEvent] // queue NID -> PDU - FederationEDUs Cache[int64, *gomatrixserverlib.EDU] // queue NID -> EDU - SpaceSummaryRooms Cache[string, gomatrixserverlib.MSC2946SpacesResponse] // room ID -> space response - LazyLoading Cache[lazyLoadingCacheKey, string] // composite key -> event ID + RoomVersions Cache[string, gomatrixserverlib.RoomVersion] // room ID -> room version + ServerKeys Cache[string, gomatrixserverlib.PublicKeyLookupResult] // server name -> server keys + RoomServerRoomNIDs Cache[string, types.RoomNID] // room ID -> room NID + RoomServerRoomIDs Cache[types.RoomNID, string] // room NID -> room ID + RoomServerEvents Cache[int64, *gomatrixserverlib.Event] // event NID -> event + RoomServerStateKeys Cache[types.EventStateKeyNID, string] // event NID -> event state key + RoomInfos Cache[string, *types.RoomInfo] // room ID -> room info + FederationPDUs Cache[int64, *gomatrixserverlib.HeaderedEvent] // queue NID -> PDU + FederationEDUs Cache[int64, *gomatrixserverlib.EDU] // queue NID -> EDU + SpaceSummaryRooms Cache[string, gomatrixserverlib.MSC2946SpacesResponse] // room ID -> space response + LazyLoading Cache[lazyLoadingCacheKey, string] // composite key -> event ID } // Cache is the interface that an implementation must satisfy. @@ -44,7 +45,7 @@ type Cache[K keyable, T any] interface { type keyable interface { // from https://github.com/dgraph-io/ristretto/blob/8e850b710d6df0383c375ec6a7beae4ce48fc8d5/z/z.go#L34 - uint64 | string | []byte | byte | int | int32 | uint32 | int64 | lazyLoadingCacheKey + ~uint64 | ~string | []byte | byte | ~int | ~int32 | ~uint32 | ~int64 | lazyLoadingCacheKey } type costable interface { diff --git a/internal/caching/impl_ristretto.go b/internal/caching/impl_ristretto.go index 01c97ad55..fdbbb4d72 100644 --- a/internal/caching/impl_ristretto.go +++ b/internal/caching/impl_ristretto.go @@ -40,6 +40,7 @@ const ( federationEDUsCache spaceSummaryRoomsCache lazyLoadingCache + eventStateKeyCache ) func NewRistrettoCache(maxCost config.DataUnit, maxAge time.Duration, enablePrometheus bool) *Caches { @@ -88,7 +89,7 @@ func NewRistrettoCache(maxCost config.DataUnit, maxAge time.Duration, enableProm Prefix: roomNIDsCache, MaxAge: maxAge, }, - RoomServerRoomIDs: &RistrettoCachePartition[int64, string]{ // room NID -> room ID + RoomServerRoomIDs: &RistrettoCachePartition[types.RoomNID, string]{ // room NID -> room ID cache: cache, Prefix: roomIDsCache, MaxAge: maxAge, @@ -100,6 +101,11 @@ func NewRistrettoCache(maxCost config.DataUnit, maxAge time.Duration, enableProm MaxAge: maxAge, }, }, + RoomServerStateKeys: &RistrettoCachePartition[types.EventStateKeyNID, string]{ // event NID -> event state key + cache: cache, + Prefix: eventStateKeyCache, + MaxAge: maxAge, + }, RoomInfos: &RistrettoCachePartition[string, *types.RoomInfo]{ // room ID -> room info cache: cache, Prefix: roomInfosCache, diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index d8d5f67c8..2f9932ff8 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -72,7 +72,24 @@ func (d *Database) eventTypeNIDs( func (d *Database) EventStateKeys( ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID, ) (map[types.EventStateKeyNID]string, error) { - return d.EventStateKeysTable.BulkSelectEventStateKey(ctx, nil, eventStateKeyNIDs) + result := make(map[types.EventStateKeyNID]string, len(eventStateKeyNIDs)) + fetch := make([]types.EventStateKeyNID, 0, len(eventStateKeyNIDs)) + for _, nid := range eventStateKeyNIDs { + if key, ok := d.Cache.GetEventStateKey(nid); ok { + result[nid] = key + } else { + fetch = append(fetch, nid) + } + } + fromDB, err := d.EventStateKeysTable.BulkSelectEventStateKey(ctx, nil, fetch) + if err != nil { + return nil, err + } + for nid, key := range fromDB { + result[nid] = key + d.Cache.StoreEventStateKey(nid, key) + } + return result, nil } func (d *Database) EventStateKeyNIDs( From 84a779788360101a2353679cc2baa695fba7f9f2 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 19 Jul 2022 17:19:03 +0300 Subject: [PATCH 17/19] Explain how SRV works in Matrix and discourage using it (#2577) * Explain how SRV works in Matrix and discourage using it * Minor tweaks to formatting Co-authored-by: Neil Alexander --- docs/installation/2_domainname.md | 60 +++++++++++++++++++------------ 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/docs/installation/2_domainname.md b/docs/installation/2_domainname.md index 0d4300eca..54064acb4 100644 --- a/docs/installation/2_domainname.md +++ b/docs/installation/2_domainname.md @@ -14,15 +14,18 @@ that take the format `@user:example.com`. For federation to work, the server name must be resolvable by other homeservers on the internet — that is, the domain must be registered and properly configured with the relevant DNS records. -Matrix servers discover each other when federating using the following methods: +Matrix servers usually discover each other when federating using the following methods: -1. If a well-known delegation exists on `example.com`, use the path server from the +1. If a well-known delegation exists on `example.com`, use the domain and port from the well-known file to connect to the remote homeserver; -2. If a DNS SRV delegation exists on `example.com`, use the hostname and port from the DNS SRV +2. If a DNS SRV delegation exists on `example.com`, use the IP address and port from the DNS SRV record to connect to the remote homeserver; 3. If neither well-known or DNS SRV delegation are configured, attempt to connect to the remote homeserver by connecting to `example.com` port TCP/8448 using HTTPS. +The exact details of how server name resolution works can be found in +[the spec](https://spec.matrix.org/v1.3/server-server-api/#resolving-server-names). + ## TLS certificates Matrix federation requires that valid TLS certificates are present on the domain. You must @@ -51,17 +54,12 @@ you will be able to delegate from `example.com` to `matrix.example.com` so that Delegation can be performed in one of two ways: -* **Well-known delegation**: A well-known text file is served over HTTPS on the domain name - that you want to use, pointing to your server on `matrix.example.com` port 8448; -* **DNS SRV delegation**: A DNS SRV record is created on the domain name that you want to - use, pointing to your server on `matrix.example.com` port TCP/8448. +* **Well-known delegation (preferred)**: A well-known text file is served over HTTPS on the domain + name that you want to use, pointing to your server on `matrix.example.com` port 8448; +* **DNS SRV delegation (not recommended)**: See the SRV delegation section below for details. -If you are using a reverse proxy to forward `/_matrix` to Dendrite, your well-known or DNS SRV -delegation must refer to the hostname and port that the reverse proxy is listening on instead. - -Well-known delegation is typically easier to set up and usually preferred. However, you can use -either or both methods to delegate. If you configure both methods of delegation, it is important -that they both agree and refer to the same hostname and port. +If you are using a reverse proxy to forward `/_matrix` to Dendrite, your well-known or delegation +must refer to the hostname and port that the reverse proxy is listening on instead. ## Well-known delegation @@ -74,20 +72,36 @@ and contain the following JSON document: ```json { - "m.server": "https://matrix.example.com:8448" + "m.server": "matrix.example.com:8448" } ``` +You can also serve `.well-known` with Dendrite itself by setting the `well_known_server_name` config +option to the value you want for `m.server`. This is primarily useful if Dendrite is exposed on +`example.com:443` and you don't want to set up a separate webserver just for serving the `.well-known` +file. + +```yaml +global: +... + well_known_server_name: "example.com:443" +``` + ## DNS SRV delegation -Using DNS SRV delegation requires creating DNS SRV records on the `example.com` zone which -refer to your Dendrite installation. +This method is not recommended, as the behavior of SRV records in Matrix is rather unintuitive: +SRV records will only change the IP address and port that other servers connect to, they won't +affect the domain name. In technical terms, the `Host` header and TLS SNI of federation requests +will still be `example.com` even if the SRV record points at `matrix.example.com`. -Assuming that your Dendrite installation is listening for HTTPS connections at `matrix.example.com` -port 8448, the DNS SRV record must have the following fields: +In practice, this means that the server must be configured with valid TLS certificates for +`example.com`, rather than `matrix.example.com` as one might intuitively expect. If there's a +reverse proxy in between, the proxy configuration must be written as if it's `example.com`, as the +proxy will never see the name `matrix.example.com` in incoming requests. -* Name: `@` (or whichever term your DNS provider uses to signal the root) -* Service: `_matrix` -* Protocol: `_tcp` -* Port: `8448` -* Target: `matrix.example.com` +This behavior also means that if `example.com` and `matrix.example.com` point at the same IP +address, there is no reason to have a SRV record pointing at `matrix.example.com`. It can still +be used to change the port number, but it won't do anything else. + +If you understand how SRV records work and still want to use them, the service name is `_matrix` and +the protocol is `_tcp`. From 9507966ebdbe0615e8d27be4b7a1d4440ada3e73 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 20 Jul 2022 12:39:06 +0200 Subject: [PATCH 18/19] Fix issue with membership event_nid being 0 (#2580) --- roomserver/storage/shared/membership_updater.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/roomserver/storage/shared/membership_updater.go b/roomserver/storage/shared/membership_updater.go index ebfcef569..c79ca8995 100644 --- a/roomserver/storage/shared/membership_updater.go +++ b/roomserver/storage/shared/membership_updater.go @@ -105,8 +105,15 @@ func (u *MembershipUpdater) SetToInvite(event *gomatrixserverlib.Event) (bool, e if err != nil { return fmt.Errorf("u.d.InvitesTable.InsertInviteEvent: %w", err) } + + // Look up the NID of the invite event + nIDs, err := u.d.eventNIDs(u.ctx, u.txn, []string{event.EventID()}, false) + if err != nil { + return fmt.Errorf("u.d.EventNIDs: %w", err) + } + if u.membership != tables.MembershipStateInvite { - if inserted, err = u.d.MembershipTable.UpdateMembership(u.ctx, u.txn, u.roomNID, u.targetUserNID, senderUserNID, tables.MembershipStateInvite, 0, false); err != nil { + if inserted, err = u.d.MembershipTable.UpdateMembership(u.ctx, u.txn, u.roomNID, u.targetUserNID, senderUserNID, tables.MembershipStateInvite, nIDs[event.EventID()], false); err != nil { return fmt.Errorf("u.d.MembershipTable.UpdateMembership: %w", err) } } From 35ce551c8ffaef4d4bbfc5a4a8fd41a8b8ef2b27 Mon Sep 17 00:00:00 2001 From: Matt Holt Date: Wed, 20 Jul 2022 23:08:17 -0600 Subject: [PATCH 19/19] docs: Add build page; correct proxy info; fix Caddy example (#2579) * Add build page; correct proxy info; fix Caddy example * Improve Caddyfile example * Apply review comments; add polylith Caddyfile --- docs/CONTRIBUTING.md | 2 +- docs/caddy/monolith/CaddyFile | 68 ------------------- docs/caddy/monolith/Caddyfile | 57 ++++++++++++++++ docs/caddy/polylith/Caddyfile | 66 ++++++++++++++++++ ...ng_polylith.md => 10_starting_polylith.md} | 2 +- ...{10_optimisation.md => 11_optimisation.md} | 2 +- docs/installation/1_planning.md | 11 +-- docs/installation/2_domainname.md | 34 +++++++--- docs/installation/3_build.md | 38 +++++++++++ .../{3_database.md => 4_database.md} | 4 +- docs/installation/6_install_polylith.md | 5 +- docs/installation/7_configuration.md | 6 +- .../{4_signingkey.md => 8_signingkey.md} | 2 +- ...ing_monolith.md => 9_starting_monolith.md} | 5 +- 14 files changed, 209 insertions(+), 93 deletions(-) delete mode 100644 docs/caddy/monolith/CaddyFile create mode 100644 docs/caddy/monolith/Caddyfile create mode 100644 docs/caddy/polylith/Caddyfile rename docs/installation/{9_starting_polylith.md => 10_starting_polylith.md} (99%) rename docs/installation/{10_optimisation.md => 11_optimisation.md} (99%) create mode 100644 docs/installation/3_build.md rename docs/installation/{3_database.md => 4_database.md} (96%) rename docs/installation/{4_signingkey.md => 8_signingkey.md} (99%) rename docs/installation/{8_starting_monolith.md => 9_starting_monolith.md} (83%) diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 5a89e6841..169224b9e 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -24,7 +24,7 @@ Unfortunately we can't accept contributions without it. ## Getting up and running -See the [Installation](INSTALL.md) section for information on how to build an +See the [Installation](installation) section for information on how to build an instance of Dendrite. You will likely need this in order to test your changes. ## Code style diff --git a/docs/caddy/monolith/CaddyFile b/docs/caddy/monolith/CaddyFile deleted file mode 100644 index cd93f9e10..000000000 --- a/docs/caddy/monolith/CaddyFile +++ /dev/null @@ -1,68 +0,0 @@ -{ - # debug - admin off - email example@example.com - default_sni example.com - # Debug endpoint - # acme_ca https://acme-staging-v02.api.letsencrypt.org/directory -} - -####################################################################### -# Snippets -#______________________________________________________________________ - -(handle_errors_maintenance) { - handle_errors { - @maintenance expression {http.error.status_code} == 502 - rewrite @maintenance maintenance.html - root * "/path/to/service/pages" - file_server - } -} - -(matrix-well-known-header) { - # Headers - header Access-Control-Allow-Origin "*" - header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS" - header Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept, Authorization" - header Content-Type "application/json" -} - -####################################################################### - -example.com { - - # ... - - handle /.well-known/matrix/server { - import matrix-well-known-header - respond `{ "m.server": "matrix.example.com:443" }` 200 - } - - handle /.well-known/matrix/client { - import matrix-well-known-header - respond `{ "m.homeserver": { "base_url": "https://matrix.example.com" } }` 200 - } - - import handle_errors_maintenance -} - -example.com:8448 { - # server<->server HTTPS traffic - reverse_proxy http://dendrite-host:8008 -} - -matrix.example.com { - - handle /_matrix/* { - # client<->server HTTPS traffic - reverse_proxy http://dendrite-host:8008 - } - - handle_path /* { - # Client webapp (Element SPA or ...) - file_server { - root /path/to/www/example.com/matrix-web-client/ - } - } -} diff --git a/docs/caddy/monolith/Caddyfile b/docs/caddy/monolith/Caddyfile new file mode 100644 index 000000000..82567c4a6 --- /dev/null +++ b/docs/caddy/monolith/Caddyfile @@ -0,0 +1,57 @@ +# Sample Caddyfile for using Caddy in front of Dendrite. +# +# Customize email address and domain names. +# Optional settings commented out. +# +# BE SURE YOUR DOMAINS ARE POINTED AT YOUR SERVER FIRST. +# Documentation: https://caddyserver.com/docs/ +# +# Bonus tip: If your IP address changes, use Caddy's +# dynamic DNS plugin to update your DNS records to +# point to your new IP automatically: +# https://github.com/mholt/caddy-dynamicdns +# + + +# Global options block +{ + # In case there is a problem with your certificates. + # email example@example.com + + # Turn off the admin endpoint if you don't need graceful config + # changes and/or are running untrusted code on your machine. + # admin off + + # Enable this if your clients don't send ServerName in TLS handshakes. + # default_sni example.com + + # Enable debug mode for verbose logging. + # debug + + # Use Let's Encrypt's staging endpoint for testing. + # acme_ca https://acme-staging-v02.api.letsencrypt.org/directory + + # If you're port-forwarding HTTP/HTTPS ports from 80/443 to something + # else, enable these and put the alternate port numbers here. + # http_port 8080 + # https_port 8443 +} + +# The server name of your matrix homeserver. This example shows +# "well-known delegation" from the registered domain to a subdomain, +# which is only needed if your server_name doesn't match your Matrix +# homeserver URL (i.e. you can show users a vanity domain that looks +# nice and is easy to remember but still have your Matrix server on +# its own subdomain or hosted service). +example.com { + header /.well-known/matrix/* Content-Type application/json + header /.well-known/matrix/* Access-Control-Allow-Origin * + respond /.well-known/matrix/server `{"m.server": "matrix.example.com:443"}` + respond /.well-known/matrix/client `{"m.homeserver": {"base_url": "https://matrix.example.com"}}` +} + +# The actual domain name whereby your Matrix server is accessed. +matrix.example.com { + # Set localhost:8008 to the address of your Dendrite server, if different + reverse_proxy /_matrix/* localhost:8008 +} diff --git a/docs/caddy/polylith/Caddyfile b/docs/caddy/polylith/Caddyfile new file mode 100644 index 000000000..244e50e7e --- /dev/null +++ b/docs/caddy/polylith/Caddyfile @@ -0,0 +1,66 @@ +# Sample Caddyfile for using Caddy in front of Dendrite. +# +# Customize email address and domain names. +# Optional settings commented out. +# +# BE SURE YOUR DOMAINS ARE POINTED AT YOUR SERVER FIRST. +# Documentation: https://caddyserver.com/docs/ +# +# Bonus tip: If your IP address changes, use Caddy's +# dynamic DNS plugin to update your DNS records to +# point to your new IP automatically: +# https://github.com/mholt/caddy-dynamicdns +# + + +# Global options block +{ + # In case there is a problem with your certificates. + # email example@example.com + + # Turn off the admin endpoint if you don't need graceful config + # changes and/or are running untrusted code on your machine. + # admin off + + # Enable this if your clients don't send ServerName in TLS handshakes. + # default_sni example.com + + # Enable debug mode for verbose logging. + # debug + + # Use Let's Encrypt's staging endpoint for testing. + # acme_ca https://acme-staging-v02.api.letsencrypt.org/directory + + # If you're port-forwarding HTTP/HTTPS ports from 80/443 to something + # else, enable these and put the alternate port numbers here. + # http_port 8080 + # https_port 8443 +} + +# The server name of your matrix homeserver. This example shows +# "well-known delegation" from the registered domain to a subdomain, +# which is only needed if your server_name doesn't match your Matrix +# homeserver URL (i.e. you can show users a vanity domain that looks +# nice and is easy to remember but still have your Matrix server on +# its own subdomain or hosted service). +example.com { + header /.well-known/matrix/* Content-Type application/json + header /.well-known/matrix/* Access-Control-Allow-Origin * + respond /.well-known/matrix/server `{"m.server": "matrix.example.com:443"}` + respond /.well-known/matrix/client `{"m.homeserver": {"base_url": "https://matrix.example.com"}}` +} + +# The actual domain name whereby your Matrix server is accessed. +matrix.example.com { + # Change the end of each reverse_proxy line to the correct + # address for your various services. + @sync_api { + path_regexp /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages)$ + } + reverse_proxy @sync_api sync_api:8073 + + reverse_proxy /_matrix/client* client_api:8071 + reverse_proxy /_matrix/federation* federation_api:8071 + reverse_proxy /_matrix/key* federation_api:8071 + reverse_proxy /_matrix/media* media_api:8071 +} diff --git a/docs/installation/9_starting_polylith.md b/docs/installation/10_starting_polylith.md similarity index 99% rename from docs/installation/9_starting_polylith.md rename to docs/installation/10_starting_polylith.md index 228e52e85..0c2e2af2b 100644 --- a/docs/installation/9_starting_polylith.md +++ b/docs/installation/10_starting_polylith.md @@ -2,7 +2,7 @@ title: Starting the polylith parent: Installation has_toc: true -nav_order: 9 +nav_order: 10 permalink: /installation/start/polylith --- diff --git a/docs/installation/10_optimisation.md b/docs/installation/11_optimisation.md similarity index 99% rename from docs/installation/10_optimisation.md rename to docs/installation/11_optimisation.md index c19b7a75e..f2f67c947 100644 --- a/docs/installation/10_optimisation.md +++ b/docs/installation/11_optimisation.md @@ -2,7 +2,7 @@ title: Optimise your installation parent: Installation has_toc: true -nav_order: 10 +nav_order: 11 permalink: /installation/start/optimisation --- diff --git a/docs/installation/1_planning.md b/docs/installation/1_planning.md index d4f3d7052..3aa5b4d85 100644 --- a/docs/installation/1_planning.md +++ b/docs/installation/1_planning.md @@ -95,12 +95,13 @@ enabled. To do so, follow the [NATS Server installation instructions](https://docs.nats.io/running-a-nats-service/introduction/installation) and then [start your NATS deployment](https://docs.nats.io/running-a-nats-service/introduction/running). JetStream must be enabled, either by passing the `-js` flag to `nats-server`, or by specifying the `store_dir` option in the the `jetstream` configuration. -### Reverse proxy (polylith deployments) +### Reverse proxy -Polylith deployments require a reverse proxy, such as [NGINX](https://www.nginx.com) or -[HAProxy](http://www.haproxy.org). Configuring those is not covered in this documentation, -although a [sample configuration for NGINX](https://github.com/matrix-org/dendrite/blob/main/docs/nginx/polylith-sample.conf) -is provided. +A reverse proxy such as [Caddy](https://caddyserver.com), [NGINX](https://www.nginx.com) or +[HAProxy](http://www.haproxy.org) is required for polylith deployments and is useful for monolith +deployments. Configuring those is not covered in this documentation, although sample configurations +for [Caddy](https://github.com/matrix-org/dendrite/blob/main/docs/caddy) and +[NGINX](https://github.com/matrix-org/dendrite/blob/main/docs/nginx) are provided. ### Windows diff --git a/docs/installation/2_domainname.md b/docs/installation/2_domainname.md index 54064acb4..7d7fc86bd 100644 --- a/docs/installation/2_domainname.md +++ b/docs/installation/2_domainname.md @@ -29,15 +29,23 @@ The exact details of how server name resolution works can be found in ## TLS certificates Matrix federation requires that valid TLS certificates are present on the domain. You must -obtain certificates from a publicly accepted Certificate Authority (CA). [LetsEncrypt](https://letsencrypt.org) -is an example of such a CA that can be used. Self-signed certificates are not suitable for -federation and will typically not be accepted by other homeservers. +obtain certificates from a publicly-trusted certificate authority (CA). [Let's Encrypt](https://letsencrypt.org) +is a popular choice of CA because the certificates are publicly-trusted, free, and automated +via the ACME protocol. (Self-signed certificates are not suitable for federation and will typically +not be accepted by other homeservers.) -A common practice to help ease the management of certificates is to install a reverse proxy in -front of Dendrite which manages the TLS certificates and HTTPS proxying itself. Software such as -[NGINX](https://www.nginx.com) and [HAProxy](http://www.haproxy.org) can be used for the task. -Although the finer details of configuring these are not described here, you must reverse proxy -all `/_matrix` paths to your Dendrite server. +Automating the renewal of TLS certificates is best practice. There are many tools for this, +but the simplest way to achieve TLS automation is to have your reverse proxy do it for you. +[Caddy](https://caddyserver.com) is recommended as a production-grade reverse proxy with +automatic TLS which is commonly used in front of Dendrite. It obtains and renews TLS certificates +automatically and by default as long as your domain name is pointed at your server first. +Although the finer details of [configuring Caddy](https://caddyserver.com/docs/) is not described +here, in general, you must reverse proxy all `/_matrix` paths to your Dendrite server. For example, +with Caddy: + +``` +reverse_proxy /_matrix/* localhost:8008 +``` It is possible for the reverse proxy to listen on the standard HTTPS port TCP/443 so long as your domain delegation is configured to point to port TCP/443. @@ -76,6 +84,16 @@ and contain the following JSON document: } ``` +For example, this can be done with the following Caddy config: + +``` +handle /.well-known/matrix/client { + header Content-Type application/json + header Access-Control-Allow-Origin * + respond `{"m.homeserver": {"base_url": "https://matrix.example.com:8448"}}` +} +``` + You can also serve `.well-known` with Dendrite itself by setting the `well_known_server_name` config option to the value you want for `m.server`. This is primarily useful if Dendrite is exposed on `example.com:443` and you don't want to set up a separate webserver just for serving the `.well-known` diff --git a/docs/installation/3_build.md b/docs/installation/3_build.md new file mode 100644 index 000000000..aed2080db --- /dev/null +++ b/docs/installation/3_build.md @@ -0,0 +1,38 @@ +--- +title: Building Dendrite +parent: Installation +has_toc: true +nav_order: 3 +permalink: /installation/build +--- + +# Build all Dendrite commands + +Dendrite has numerous utility commands in addition to the actual server binaries. +Build them all from the root of the source repo with `build.sh` (Linux/Mac): + +```sh +./build.sh +``` + +or `build.cmd` (Windows): + +```powershell +build.cmd +``` + +The resulting binaries will be placed in the `bin` subfolder. + +# Installing as a monolith + +You can install the Dendrite monolith binary into `$GOPATH/bin` by using `go install`: + +```sh +go install ./cmd/dendrite-monolith-server +``` + +Alternatively, you can specify a custom path for the binary to be written to using `go build`: + +```sh +go build -o /usr/local/bin/ ./cmd/dendrite-monolith-server +``` diff --git a/docs/installation/3_database.md b/docs/installation/4_database.md similarity index 96% rename from docs/installation/3_database.md rename to docs/installation/4_database.md index f64fe9150..f6222a8d2 100644 --- a/docs/installation/3_database.md +++ b/docs/installation/4_database.md @@ -17,7 +17,9 @@ filenames in the Dendrite configuration file and start Dendrite. The databases w and populated automatically. Note that Dendrite **cannot share a single SQLite database across multiple components**. Each -component must be configured with its own SQLite database filename. +component must be configured with its own SQLite database filename. You will have to remove +the `global.database` section from your Dendrite config and add it to each individual section +instead in order to use SQLite. ### Connection strings diff --git a/docs/installation/6_install_polylith.md b/docs/installation/6_install_polylith.md index 375512f8f..ec4a77628 100644 --- a/docs/installation/6_install_polylith.md +++ b/docs/installation/6_install_polylith.md @@ -29,5 +29,6 @@ Polylith deployments require a reverse proxy in order to ensure that requests ar sent to the correct endpoint. You must ensure that a suitable reverse proxy is installed and configured. -A [sample configuration file](https://github.com/matrix-org/dendrite/blob/main/docs/nginx/polylith-sample.conf) -is provided for [NGINX](https://www.nginx.com). +Sample configurations are provided +for [Caddy](https://github.com/matrix-org/dendrite/blob/main/docs/caddy/polylith/Caddyfile) +and [NGINX](https://github.com/matrix-org/dendrite/blob/main/docs/nginx/polylith-sample.conf). \ No newline at end of file diff --git a/docs/installation/7_configuration.md b/docs/installation/7_configuration.md index e676afbe6..b1c747414 100644 --- a/docs/installation/7_configuration.md +++ b/docs/installation/7_configuration.md @@ -1,13 +1,13 @@ --- -title: Populate the configuration +title: Configuring Dendrite parent: Installation nav_order: 7 permalink: /installation/configuration --- -# Populate the configuration +# Configuring Dendrite -The configuration file is used to configure Dendrite. Sample configuration files are +A YAML configuration file is used to configure Dendrite. Sample configuration files are present in the top level of the Dendrite repository: * [`dendrite-sample.monolith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.monolith.yaml) diff --git a/docs/installation/4_signingkey.md b/docs/installation/8_signingkey.md similarity index 99% rename from docs/installation/4_signingkey.md rename to docs/installation/8_signingkey.md index 07dc485ff..323759a88 100644 --- a/docs/installation/4_signingkey.md +++ b/docs/installation/8_signingkey.md @@ -1,7 +1,7 @@ --- title: Generating signing keys parent: Installation -nav_order: 4 +nav_order: 8 permalink: /installation/signingkeys --- diff --git a/docs/installation/8_starting_monolith.md b/docs/installation/9_starting_monolith.md similarity index 83% rename from docs/installation/8_starting_monolith.md rename to docs/installation/9_starting_monolith.md index e0e7309d2..124477e73 100644 --- a/docs/installation/8_starting_monolith.md +++ b/docs/installation/9_starting_monolith.md @@ -15,8 +15,9 @@ you can start your Dendrite monolith deployment by starting the `dendrite-monoli ./dendrite-monolith-server -config /path/to/dendrite.yaml ``` -If you want to change the addresses or ports that Dendrite listens on, you -can use the `-http-bind-address` and `-https-bind-address` command line arguments: +By default, Dendrite will listen HTTP on port 8008. If you want to change the addresses +or ports that Dendrite listens on, you can use the `-http-bind-address` and +`-https-bind-address` command line arguments: ```bash ./dendrite-monolith-server -config /path/to/dendrite.yaml \