mirrors/dendrite
1
0
Fork 0
mirror of https://github.com/matrix-org/dendrite.git synced 2026-01-10 23:53:09 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into s7evink/expireedus

Browse source
This commit is contained in:
Neil Alexander 2022-08-05 10:17:57 +01:00 committed by GitHub
parent d0f15d3aef 1b7f84250a
commit fa38bf2df8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
46 changed files with 313 additions and 253 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
.github/workflows/dendrite.yml vendored
View file

@ -97,7 +97,7 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
go: ["1.18"] go: ["1.18", "1.19"]
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- name: Setup go - name: Setup go
@ -127,7 +127,7 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
go: ["1.18"] go: ["1.18", "1.19"]
goos: ["linux"] goos: ["linux"]
goarch: ["amd64", "386"] goarch: ["amd64", "386"]
steps: steps:
@ -151,6 +151,7 @@ jobs:
GOOS: ${{ matrix.goos }} GOOS: ${{ matrix.goos }}
GOARCH: ${{ matrix.goarch }} GOARCH: ${{ matrix.goarch }}
CGO_ENABLED: 1 CGO_ENABLED: 1
CGO_CFLAGS: -fno-stack-protector
run: go build -trimpath -v -o "bin/" ./cmd/... run: go build -trimpath -v -o "bin/" ./cmd/...
# build for Windows 64-bit # build for Windows 64-bit
@ -160,7 +161,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
strategy: strategy:
matrix: matrix:
go: ["1.18"] go: ["1.18", "1.19"]
goos: ["windows"] goos: ["windows"]
goarch: ["amd64"] goarch: ["amd64"]
steps: steps:
@ -384,7 +385,14 @@ jobs:
integration-tests-done: integration-tests-done:
name: Integration tests passed name: Integration tests passed
needs: [initial-tests-done, upgrade_test, upgrade_test_direct, sytest, complement] needs:
[
initial-tests-done,
upgrade_test,
upgrade_test_direct,
sytest,
complement,
]
runs-on: ubuntu-latest runs-on: ubuntu-latest
if: ${{ !cancelled() }} # Run this even if prior jobs were skipped if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
steps: steps:

14
CHANGES.md
View file

@ -1,5 +1,19 @@
# Changelog # Changelog
## Dendrite 0.9.1 (2022-08-03)
### Fixes
* Upgrades a dependency which caused issues building Dendrite with Go 1.19
* The roomserver will no longer give up prematurely after failing to call `/state_ids`
* Removes the faulty room info cache, which caused of a number of race conditions and occasional bugs (including when creating and joining rooms)
* The media endpoint now sets the `Cache-Control` header correctly to prevent web-based clients from hitting media endpoints excessively
* The sync API will now advance the PDU stream position correctly in all cases (contributed by [sergekh2](https://github.com/sergekh2))
* The sync API will now delete the correct range of send-to-device messages when advancing the stream position
* The device list `changed` key in the `/sync` response should now return the correct users
* A data race when looking up missing state has been fixed
* The `/send_join` API is now applying stronger validation to the received membership event
## Dendrite 0.9.0 (2022-08-01) ## Dendrite 0.9.0 (2022-08-01)
### Features ### Features

2
build/scripts/build-test-lint.sh
View file

@ -13,4 +13,4 @@ go build ./cmd/...
./build/scripts/find-lint.sh ./build/scripts/find-lint.sh
echo "Testing..." echo "Testing..."
go test -v ./... go test --race -v ./...

5
clientapi/routing/directory_public.go
View file

@ -23,13 +23,14 @@ import (
"strings" "strings"
"sync" "sync"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/matrix-org/dendrite/clientapi/api" "github.com/matrix-org/dendrite/clientapi/api"
"github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/jsonerror"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
) )
var ( var (

1
clientapi/routing/sendevent.go
View file

@ -63,6 +63,7 @@ var sendEventDuration = prometheus.NewHistogramVec(
) )
// SendEvent implements: // SendEvent implements:
//
// /rooms/{roomID}/send/{eventType} // /rooms/{roomID}/send/{eventType}
// /rooms/{roomID}/send/{eventType}/{txnID} // /rooms/{roomID}/send/{eventType}/{txnID}
// /rooms/{roomID}/state/{eventType}/{stateKey} // /rooms/{roomID}/state/{eventType}/{stateKey}

1
clientapi/routing/threepid.go
View file

@ -38,6 +38,7 @@ type threePIDsResponse struct {
} }
// RequestEmailToken implements: // RequestEmailToken implements:
//
// POST /account/3pid/email/requestToken // POST /account/3pid/email/requestToken
// POST /register/email/requestToken // POST /register/email/requestToken
func RequestEmailToken(req *http.Request, threePIDAPI api.ClientUserAPI, cfg *config.ClientAPI) util.JSONResponse { func RequestEmailToken(req *http.Request, threePIDAPI api.ClientUserAPI, cfg *config.ClientAPI) util.JSONResponse {

6
clientapi/routing/voip.go
View file

@ -22,14 +22,16 @@ import (
"net/http" "net/http"
"time" "time"
"github.com/matrix-org/gomatrix"
"github.com/matrix-org/util"
"github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrix"
"github.com/matrix-org/util"
) )
// RequestTurnServer implements: // RequestTurnServer implements:
//
// GET /voip/turnServer // GET /voip/turnServer
func RequestTurnServer(req *http.Request, device *api.Device, cfg *config.ClientAPI) util.JSONResponse { func RequestTurnServer(req *http.Request, device *api.Device, cfg *config.ClientAPI) util.JSONResponse {
turnConfig := cfg.TURN turnConfig := cfg.TURN

2
cmd/dendrite-upgrade-tests/main.go
View file

@ -95,7 +95,9 @@ CMD /build/run_dendrite.sh `
const dendriteUpgradeTestLabel = "dendrite_upgrade_test" const dendriteUpgradeTestLabel = "dendrite_upgrade_test"
// downloadArchive downloads an arbitrary github archive of the form: // downloadArchive downloads an arbitrary github archive of the form:
//
// https://github.com/matrix-org/dendrite/archive/v0.3.11.tar.gz // https://github.com/matrix-org/dendrite/archive/v0.3.11.tar.gz
//
// and re-tarballs it without the top-level directory which contains branch information. It inserts // and re-tarballs it without the top-level directory which contains branch information. It inserts
// the contents of `dockerfile` as a root file `Dockerfile` in the re-tarballed directory such that // the contents of `dockerfile` as a root file `Dockerfile` in the re-tarballed directory such that
// you can directly feed the retarballed archive to `ImageBuild` to have it run said dockerfile. // you can directly feed the retarballed archive to `ImageBuild` to have it run said dockerfile.

2
dendrite-sample.monolith.yaml
View file

@ -192,7 +192,7 @@ client_api:
# and appservice users are exempt from rate limiting by default. # and appservice users are exempt from rate limiting by default.
rate_limiting: rate_limiting:
enabled: true enabled: true
threshold: 5 threshold: 20
cooloff_ms: 500 cooloff_ms: 500
exempt_user_ids: exempt_user_ids:
# - "@user:domain.com" # - "@user:domain.com"

2
dendrite-sample.polylith.yaml
View file

@ -195,7 +195,7 @@ client_api:
# and appservice users are exempt from rate limiting by default. # and appservice users are exempt from rate limiting by default.
rate_limiting: rate_limiting:
enabled: true enabled: true
threshold: 5 threshold: 20
cooloff_ms: 500 cooloff_ms: 500
exempt_user_ids: exempt_user_ids:
# - "@user:domain.com" # - "@user:domain.com"

2
docs/CONTRIBUTING.md
View file

@ -64,7 +64,7 @@ comment. Please avoid doing this if you can.
We also have unit tests which we run via: We also have unit tests which we run via:
```bash ```bash
go test ./... go test --race ./...
``` ```
In general, we like submissions that come with tests. Anything that proves that the In general, we like submissions that come with tests. Anything that proves that the

8
federationapi/consumers/roomserver.go
View file

@ -208,9 +208,11 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew
// joinedHostsAtEvent works out a list of matrix servers that were joined to // joinedHostsAtEvent works out a list of matrix servers that were joined to
// the room at the event (including peeking ones) // the room at the event (including peeking ones)
// It is important to use the state at the event for sending messages because: // It is important to use the state at the event for sending messages because:
// 1) We shouldn't send messages to servers that weren't in the room. //
// 2) If a server is kicked from the rooms it should still be told about the // 1. We shouldn't send messages to servers that weren't in the room.
// kick event, // 2. If a server is kicked from the rooms it should still be told about the
// kick event.
//
// Usually the list can be calculated locally, but sometimes it will need fetch // Usually the list can be calculated locally, but sometimes it will need fetch
// events from the room server. // events from the room server.
// Returns an error if there was a problem talking to the room server. // Returns an error if there was a problem talking to the room server.

10
federationapi/federationapi_test.go
View file

@ -6,6 +6,7 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"strings" "strings"
"sync"
"testing" "testing"
"time" "time"
@ -48,6 +49,7 @@ func (f *fedRoomserverAPI) QueryRoomsForUser(ctx context.Context, req *rsapi.Que
// TODO: This struct isn't generic, only works for TestFederationAPIJoinThenKeyUpdate // TODO: This struct isn't generic, only works for TestFederationAPIJoinThenKeyUpdate
type fedClient struct { type fedClient struct {
fedClientMutex sync.Mutex
api.FederationClient api.FederationClient
allowJoins []*test.Room allowJoins []*test.Room
keys map[gomatrixserverlib.ServerName]struct { keys map[gomatrixserverlib.ServerName]struct {
@ -59,6 +61,8 @@ type fedClient struct {
} }
func (f *fedClient) GetServerKeys(ctx context.Context, matrixServer gomatrixserverlib.ServerName) (gomatrixserverlib.ServerKeys, error) { func (f *fedClient) GetServerKeys(ctx context.Context, matrixServer gomatrixserverlib.ServerName) (gomatrixserverlib.ServerKeys, error) {
f.fedClientMutex.Lock()
defer f.fedClientMutex.Unlock()
fmt.Println("GetServerKeys:", matrixServer) fmt.Println("GetServerKeys:", matrixServer)
var keys gomatrixserverlib.ServerKeys var keys gomatrixserverlib.ServerKeys
var keyID gomatrixserverlib.KeyID var keyID gomatrixserverlib.KeyID
@ -122,6 +126,8 @@ func (f *fedClient) MakeJoin(ctx context.Context, s gomatrixserverlib.ServerName
return return
} }
func (f *fedClient) SendJoin(ctx context.Context, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res gomatrixserverlib.RespSendJoin, err error) { func (f *fedClient) SendJoin(ctx context.Context, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res gomatrixserverlib.RespSendJoin, err error) {
f.fedClientMutex.Lock()
defer f.fedClientMutex.Unlock()
for _, r := range f.allowJoins { for _, r := range f.allowJoins {
if r.ID == event.RoomID() { if r.ID == event.RoomID() {
r.InsertEvent(f.t, event.Headered(r.Version)) r.InsertEvent(f.t, event.Headered(r.Version))
@ -134,6 +140,8 @@ func (f *fedClient) SendJoin(ctx context.Context, s gomatrixserverlib.ServerName
} }
func (f *fedClient) SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error) { func (f *fedClient) SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error) {
f.fedClientMutex.Lock()
defer f.fedClientMutex.Unlock()
for _, edu := range t.EDUs { for _, edu := range t.EDUs {
if edu.Type == gomatrixserverlib.MDeviceListUpdate { if edu.Type == gomatrixserverlib.MDeviceListUpdate {
f.sentTxn = true f.sentTxn = true
@ -242,6 +250,8 @@ func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) {
testrig.MustPublishMsgs(t, jsctx, msg) testrig.MustPublishMsgs(t, jsctx, msg)
time.Sleep(500 * time.Millisecond) time.Sleep(500 * time.Millisecond)
fc.fedClientMutex.Lock()
defer fc.fedClientMutex.Unlock()
if !fc.sentTxn { if !fc.sentTxn {
t.Fatalf("did not send device list update") t.Fatalf("did not send device list update")
} }

2
federationapi/queue/queue.go
View file

@ -158,7 +158,7 @@ func (oqs *OutgoingQueues) getQueue(destination gomatrixserverlib.ServerName) *d
oqs.queuesMutex.Lock() oqs.queuesMutex.Lock()
defer oqs.queuesMutex.Unlock() defer oqs.queuesMutex.Unlock()
oq, ok := oqs.queues[destination] oq, ok := oqs.queues[destination]
if !ok || oq != nil { if !ok || oq == nil {
destinationQueueTotal.Inc() destinationQueueTotal.Inc()
oq = &destinationQueue{ oq = &destinationQueue{
queues: oqs, queues: oqs,

13
federationapi/routing/join.go
View file

@ -21,13 +21,14 @@ import (
"sort" "sort"
"time" "time"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/sirupsen/logrus"
"github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/sirupsen/logrus"
) )
// MakeJoin implements the /make_join API // MakeJoin implements the /make_join API
@ -435,11 +436,11 @@ func SendJoin(
// a restricted room join. If the room version does not support restricted // a restricted room join. If the room version does not support restricted
// joins then this function returns with no side effects. This returns three // joins then this function returns with no side effects. This returns three
// values: // values:
// * an optional JSON response body (i.e. M_UNABLE_TO_AUTHORISE_JOIN) which // - an optional JSON response body (i.e. M_UNABLE_TO_AUTHORISE_JOIN) which
// should always be sent back to the client if one is specified // should always be sent back to the client if one is specified
// * a user ID of an authorising user, typically a user that has power to // - a user ID of an authorising user, typically a user that has power to
// issue invites in the room, if one has been found // issue invites in the room, if one has been found
// * an error if there was a problem finding out if this was allowable, // - an error if there was a problem finding out if this was allowable,
// like if the room version isn't known or a problem happened talking to // like if the room version isn't known or a problem happened talking to
// the roomserver // the roomserver
func checkRestrictedJoin( func checkRestrictedJoin(

11
go.mod
View file

@ -22,7 +22,7 @@ require (
github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
github.com/matrix-org/gomatrixserverlib v0.0.0-20220725104114-b6003e522771 github.com/matrix-org/gomatrixserverlib v0.0.0-20220725104114-b6003e522771
github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9
github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
github.com/mattn/go-sqlite3 v1.14.13 github.com/mattn/go-sqlite3 v1.14.13
github.com/nats-io/nats-server/v2 v2.8.5-0.20220731184415-903a06a5b4ee github.com/nats-io/nats-server/v2 v2.8.5-0.20220731184415-903a06a5b4ee
@ -45,7 +45,7 @@ require (
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e
golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9 golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9
golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd
golang.org/x/net v0.0.0-20220524220425-1d687d428aca golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e
golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 golang.org/x/term v0.0.0-20220526004731-065cf7ba2467
gopkg.in/h2non/bimg.v1 v1.1.9 gopkg.in/h2non/bimg.v1 v1.1.9
gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v2 v2.4.0
@ -73,10 +73,11 @@ require (
github.com/juju/errors v0.0.0-20220203013757-bd733f3c86b9 // indirect github.com/juju/errors v0.0.0-20220203013757-bd733f3c86b9 // indirect
github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494 // indirect github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494 // indirect
github.com/klauspost/compress v1.15.9 // indirect github.com/klauspost/compress v1.15.9 // indirect
github.com/lucas-clemente/quic-go v0.26.0 // indirect github.com/lucas-clemente/quic-go v0.28.1 // indirect
github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
github.com/marten-seemann/qtls-go1-17 v0.1.1 // indirect github.com/marten-seemann/qtls-go1-17 v0.1.2 // indirect
github.com/marten-seemann/qtls-go1-18 v0.1.1 // indirect github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect
github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/miekg/dns v1.1.49 // indirect github.com/miekg/dns v1.1.49 // indirect
github.com/minio/highwayhash v1.0.2 // indirect github.com/minio/highwayhash v1.0.2 // indirect

25
go.sum
View file

@ -321,8 +321,8 @@ github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/lib/pq v1.10.5 h1:J+gdV2cUmX7ZqL2B0lFcW0m+egaHC2V3lpO8nWxyYiQ= github.com/lib/pq v1.10.5 h1:J+gdV2cUmX7ZqL2B0lFcW0m+egaHC2V3lpO8nWxyYiQ=
github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lucas-clemente/quic-go v0.26.0 h1:ALBQXr9UJ8A1LyzvceX4jd9QFsHvlI0RR6BkV16o00A= github.com/lucas-clemente/quic-go v0.28.1 h1:Uo0lvVxWg5la9gflIF9lwa39ONq85Xq2D91YNEIslzU=
github.com/lucas-clemente/quic-go v0.26.0/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI= github.com/lucas-clemente/quic-go v0.28.1/go.mod h1:oGz5DKK41cJt5+773+BSO9BXDsREY4HLf7+0odGAPO0=
github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
github.com/lxn/walk v0.0.0-20210112085537-c389da54e794/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ= github.com/lxn/walk v0.0.0-20210112085537-c389da54e794/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ=
github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk= github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
@ -330,10 +330,12 @@ github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN
github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
github.com/marten-seemann/qtls-go1-16 v0.1.5 h1:o9JrYPPco/Nukd/HpOHMHZoBDXQqoNtUCmny98/1uqQ= github.com/marten-seemann/qtls-go1-16 v0.1.5 h1:o9JrYPPco/Nukd/HpOHMHZoBDXQqoNtUCmny98/1uqQ=
github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk= github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk=
github.com/marten-seemann/qtls-go1-17 v0.1.1 h1:DQjHPq+aOzUeh9/lixAGunn6rIOQyWChPSI4+hgW7jc= github.com/marten-seemann/qtls-go1-17 v0.1.2 h1:JADBlm0LYiVbuSySCHeY863dNkcpMmDR7s0bLKJeYlQ=
github.com/marten-seemann/qtls-go1-17 v0.1.1/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s= github.com/marten-seemann/qtls-go1-17 v0.1.2/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s=
github.com/marten-seemann/qtls-go1-18 v0.1.1 h1:qp7p7XXUFL7fpBvSS1sWD+uSqPvzNQK43DH+/qEkj0Y= github.com/marten-seemann/qtls-go1-18 v0.1.2 h1:JH6jmzbduz0ITVQ7ShevK10Av5+jBEKAHMntXmIV7kM=
github.com/marten-seemann/qtls-go1-18 v0.1.1/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= github.com/marten-seemann/qtls-go1-18 v0.1.2/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4=
github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1 h1:7m/WlWcSROrcK5NxuXaxYD32BZqe/LEEnBrWcH/cOqQ=
github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI=
github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e h1:DP5RC0Z3XdyBEW5dKt8YPeN6vZbm6OzVaGVp7f1BQRM= github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e h1:DP5RC0Z3XdyBEW5dKt8YPeN6vZbm6OzVaGVp7f1BQRM=
github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg=
github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw2QV3YD/fRrzEDPNGgTlJlvXY0EHHnT87wF3OA= github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw2QV3YD/fRrzEDPNGgTlJlvXY0EHHnT87wF3OA=
@ -343,8 +345,8 @@ github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5d
github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
github.com/matrix-org/gomatrixserverlib v0.0.0-20220725104114-b6003e522771 h1:ZIPHFIPNDS9dmEbPEiJbNmyCGJtn9exfpLC7JOcn/bE= github.com/matrix-org/gomatrixserverlib v0.0.0-20220725104114-b6003e522771 h1:ZIPHFIPNDS9dmEbPEiJbNmyCGJtn9exfpLC7JOcn/bE=
github.com/matrix-org/gomatrixserverlib v0.0.0-20220725104114-b6003e522771/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk= github.com/matrix-org/gomatrixserverlib v0.0.0-20220725104114-b6003e522771/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a h1:DdG8vXMlZ65EAtc4V+3t7zHZ2Gqs24pSnyXS+4BRHUs= github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9 h1:ed8yvWhTLk7+sNeK/eOZRTvESFTOHDRevoRoyeqPtvY=
github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc= github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9/go.mod h1:P4MqPf+u83OPulPJ+XTbSDbbWrdFYNY4LZ/B1PIduFE=
github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=
github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
@ -549,7 +551,6 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
@ -659,8 +660,8 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20210927181540-4e4d966f7476/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210927181540-4e4d966f7476/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211011170408-caeb26a5c8c0/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211011170408-caeb26a5c8c0/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211101193420-4a448f8816b3/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211101193420-4a448f8816b3/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220524220425-1d687d428aca h1:xTaFYiPROfpPhqrfTIDXj0ri1SpfueYT951s4bAuDO8= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e h1:TsQ7F31D3bUCLeqPT0u+yjp1guoArKaNKmCr22PYgTQ=
golang.org/x/net v0.0.0-20220524220425-1d687d428aca/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@ -750,6 +751,7 @@ golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM=
golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -758,6 +760,7 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b h1:NXqSWXSRUSCaFuvitrWtU169I3876zRTalMRbfd6LL0= golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b h1:NXqSWXSRUSCaFuvitrWtU169I3876zRTalMRbfd6LL0=
golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b/go.mod h1:EFNZuWvGYxIRUEX+K8UmCFwYmZjqcrnq15ZuVldZkZ0= golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b/go.mod h1:EFNZuWvGYxIRUEX+K8UmCFwYmZjqcrnq15ZuVldZkZ0=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

33
internal/caching/cache_roominfo.go
View file

@ -1,33 +0,0 @@
package caching
import (
"github.com/matrix-org/dendrite/roomserver/types"
)
// WARNING: This cache is mutable because it's entirely possible that
// the IsStub or StateSnaphotNID fields can change, even though the
// room version and room NID fields will not. This is only safe because
// the RoomInfoCache is used ONLY within the roomserver and because it
// will be kept up-to-date by the latest events updater. It MUST NOT be
// used from other components as we currently have no way to invalidate
// the cache in downstream components.
// RoomInfosCache contains the subset of functions needed for
// a room Info cache. It must only be used from the roomserver only
// It is not safe for use from other components.
type RoomInfoCache interface {
GetRoomInfo(roomID string) (roomInfo *types.RoomInfo, ok bool)
StoreRoomInfo(roomID string, roomInfo *types.RoomInfo)
}
// GetRoomInfo must only be called from the roomserver only. It is not
// safe for use from other components.
func (c Caches) GetRoomInfo(roomID string) (*types.RoomInfo, bool) {
return c.RoomInfos.Get(roomID)
}
// StoreRoomInfo must only be called from the roomserver only. It is not
// safe for use from other components.
func (c Caches) StoreRoomInfo(roomID string, roomInfo *types.RoomInfo) {
c.RoomInfos.Set(roomID, roomInfo)
}

1
internal/caching/cache_roomservernids.go
View file

@ -7,7 +7,6 @@ import (
type RoomServerCaches interface { type RoomServerCaches interface {
RoomServerNIDsCache RoomServerNIDsCache
RoomVersionCache RoomVersionCache
RoomInfoCache
RoomServerEventsCache RoomServerEventsCache
EventStateKeyCache EventStateKeyCache
} }

1
internal/caching/caches.go
View file

@ -29,7 +29,6 @@ type Caches struct {
RoomServerRoomIDs Cache[types.RoomNID, string] // room NID -> room ID RoomServerRoomIDs Cache[types.RoomNID, string] // room NID -> room ID
RoomServerEvents Cache[int64, *gomatrixserverlib.Event] // event NID -> event RoomServerEvents Cache[int64, *gomatrixserverlib.Event] // event NID -> event
RoomServerStateKeys Cache[types.EventStateKeyNID, string] // event NID -> event state key RoomServerStateKeys Cache[types.EventStateKeyNID, string] // event NID -> event state key
RoomInfos Cache[string, *types.RoomInfo] // room ID -> room info
FederationPDUs Cache[int64, *gomatrixserverlib.HeaderedEvent] // queue NID -> PDU FederationPDUs Cache[int64, *gomatrixserverlib.HeaderedEvent] // queue NID -> PDU
FederationEDUs Cache[int64, *gomatrixserverlib.EDU] // queue NID -> EDU FederationEDUs Cache[int64, *gomatrixserverlib.EDU] // queue NID -> EDU
SpaceSummaryRooms Cache[string, gomatrixserverlib.MSC2946SpacesResponse] // room ID -> space response SpaceSummaryRooms Cache[string, gomatrixserverlib.MSC2946SpacesResponse] // room ID -> space response

7
internal/caching/impl_ristretto.go
View file

@ -35,7 +35,6 @@ const (
roomNIDsCache roomNIDsCache
roomIDsCache roomIDsCache
roomEventsCache roomEventsCache
roomInfosCache
federationPDUsCache federationPDUsCache
federationEDUsCache federationEDUsCache
spaceSummaryRoomsCache spaceSummaryRoomsCache
@ -106,12 +105,6 @@ func NewRistrettoCache(maxCost config.DataUnit, maxAge time.Duration, enableProm
Prefix: eventStateKeyCache, Prefix: eventStateKeyCache,
MaxAge: maxAge, MaxAge: maxAge,
}, },
RoomInfos: &RistrettoCachePartition[string, *types.RoomInfo]{ // room ID -> room info
cache: cache,
Prefix: roomInfosCache,
Mutable: true,
MaxAge: maxAge,
},
FederationPDUs: &RistrettoCostedCachePartition[int64, *gomatrixserverlib.HeaderedEvent]{ // queue NID -> PDU FederationPDUs: &RistrettoCostedCachePartition[int64, *gomatrixserverlib.HeaderedEvent]{ // queue NID -> PDU
&RistrettoCachePartition[int64, *gomatrixserverlib.HeaderedEvent]{ &RistrettoCachePartition[int64, *gomatrixserverlib.HeaderedEvent]{
cache: cache, cache: cache,

3
internal/log.go
View file

@ -27,9 +27,10 @@ import (
"github.com/matrix-org/util" "github.com/matrix-org/util"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dugong" "github.com/matrix-org/dugong"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"github.com/matrix-org/dendrite/setup/config"
) )
type utcFormatter struct { type utcFormatter struct {

2
internal/version.go
View file

@ -17,7 +17,7 @@ var build string
const ( const (
VersionMajor = 0 VersionMajor = 0
VersionMinor = 9 VersionMinor = 9
VersionPatch = 0 VersionPatch = 1
VersionTag = "" // example: "rc1" VersionTag = "" // example: "rc1"
) )

7
keyserver/internal/device_list_update.go
View file

@ -22,12 +22,13 @@ import (
"sync" "sync"
"time" "time"
fedsenderapi "github.com/matrix-org/dendrite/federationapi/api"
"github.com/matrix-org/dendrite/keyserver/api"
"github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util" "github.com/matrix-org/util"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
fedsenderapi "github.com/matrix-org/dendrite/federationapi/api"
"github.com/matrix-org/dendrite/keyserver/api"
) )
var ( var (
@ -66,12 +67,14 @@ func init() {
// - We don't have unbounded growth in proportion to the number of servers (this is more important in a P2P world where // - We don't have unbounded growth in proportion to the number of servers (this is more important in a P2P world where
// we have many many servers) // we have many many servers)
// - We can adjust concurrency (at the cost of memory usage) by tuning N, to accommodate mobile devices vs servers. // - We can adjust concurrency (at the cost of memory usage) by tuning N, to accommodate mobile devices vs servers.
//
// The downsides are that: // The downsides are that:
// - Query requests can get queued behind other servers if they hash to the same worker, even if there are other free // - Query requests can get queued behind other servers if they hash to the same worker, even if there are other free
// workers elsewhere. Whilst suboptimal, provided we cap how long a single request can last (e.g using context timeouts) // workers elsewhere. Whilst suboptimal, provided we cap how long a single request can last (e.g using context timeouts)
// we guarantee we will get around to it. Also, more users on a given server does not increase the number of requests // we guarantee we will get around to it. Also, more users on a given server does not increase the number of requests
// (as /keys/query allows multiple users to be specified) so being stuck behind matrix.org won't materially be any worse // (as /keys/query allows multiple users to be specified) so being stuck behind matrix.org won't materially be any worse
// than being stuck behind foo.bar // than being stuck behind foo.bar
//
// In the event that the query fails, a lock is acquired and the server name along with the time to wait before retrying is // In the event that the query fails, a lock is acquired and the server name along with the time to wait before retrying is
// set in a map. A restarter goroutine periodically probes this map and injects servers which are ready to be retried. // set in a map. A restarter goroutine periodically probes this map and injects servers which are ready to be retried.
type DeviceListUpdater struct { type DeviceListUpdater struct {

3
keyserver/internal/device_list_update_test.go
View file

@ -27,8 +27,9 @@ import (
"testing" "testing"
"time" "time"
"github.com/matrix-org/dendrite/keyserver/api"
"github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/dendrite/keyserver/api"
) )
var ( var (

11
keyserver/internal/internal.go
View file

@ -18,6 +18,7 @@ import (
"bytes" "bytes"
"context" "context"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"sync" "sync"
"time" "time"
@ -314,6 +315,11 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques
for targetKeyID := range masterKey.Keys { for targetKeyID := range masterKey.Keys {
sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, req.UserID, targetUserID, targetKeyID) sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, req.UserID, targetUserID, targetKeyID)
if err != nil { if err != nil {
// Stop executing the function if the context was canceled/the deadline was exceeded,
// as we can't continue without a valid context.
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return
}
logrus.WithError(err).Errorf("a.DB.CrossSigningSigsForTarget failed") logrus.WithError(err).Errorf("a.DB.CrossSigningSigsForTarget failed")
continue continue
} }
@ -335,6 +341,11 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques
for targetKeyID, key := range forUserID { for targetKeyID, key := range forUserID {
sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, req.UserID, targetUserID, gomatrixserverlib.KeyID(targetKeyID)) sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, req.UserID, targetUserID, gomatrixserverlib.KeyID(targetKeyID))
if err != nil { if err != nil {
// Stop executing the function if the context was canceled/the deadline was exceeded,
// as we can't continue without a valid context.
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return
}
logrus.WithError(err).Errorf("a.DB.CrossSigningSigsForTarget failed") logrus.WithError(err).Errorf("a.DB.CrossSigningSigsForTarget failed")
continue continue
} }

9
keyserver/storage/storage_test.go
View file

@ -3,6 +3,7 @@ package storage_test
import ( import (
"context" "context"
"reflect" "reflect"
"sync"
"testing" "testing"
"github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/keyserver/api"
@ -103,6 +104,9 @@ func TestKeyChangesUpperLimit(t *testing.T) {
}) })
} }
var dbLock sync.Mutex
var deviceArray = []string{"AAA", "another_device"}
// The purpose of this test is to make sure that the storage layer is generating sequential stream IDs per user, // The purpose of this test is to make sure that the storage layer is generating sequential stream IDs per user,
// and that they are returned correctly when querying for device keys. // and that they are returned correctly when querying for device keys.
func TestDeviceKeysStreamIDGeneration(t *testing.T) { func TestDeviceKeysStreamIDGeneration(t *testing.T) {
@ -169,8 +173,11 @@ func TestDeviceKeysStreamIDGeneration(t *testing.T) {
t.Fatalf("Expected StoreLocalDeviceKeys to set StreamID=3 (new key same device) but got %d", msgs[0].StreamID) t.Fatalf("Expected StoreLocalDeviceKeys to set StreamID=3 (new key same device) but got %d", msgs[0].StreamID)
} }
dbLock.Lock()
defer dbLock.Unlock()
// Querying for device keys returns the latest stream IDs // Querying for device keys returns the latest stream IDs
msgs, err = db.DeviceKeysForUser(ctx, alice, []string{"AAA", "another_device"}, false) msgs, err = db.DeviceKeysForUser(ctx, alice, deviceArray, false)
if err != nil { if err != nil {
t.Fatalf("DeviceKeysForUser returned error: %s", err) t.Fatalf("DeviceKeysForUser returned error: %s", err)
} }

3
mediaapi/routing/routing.go
View file

@ -149,6 +149,9 @@ func makeDownloadAPI(
} }
} }
// Cache media for at least one day.
w.Header().Set("Cache-Control", "public,max-age=86400,s-maxage=86400")
Download( Download(
w, w,
req, req,

9
roomserver/internal/input/input.go
View file

@ -25,6 +25,11 @@ import (
"github.com/Arceliar/phony" "github.com/Arceliar/phony"
"github.com/getsentry/sentry-go" "github.com/getsentry/sentry-go"
"github.com/matrix-org/gomatrixserverlib"
"github.com/nats-io/nats.go"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
fedapi "github.com/matrix-org/dendrite/federationapi/api" fedapi "github.com/matrix-org/dendrite/federationapi/api"
"github.com/matrix-org/dendrite/roomserver/acls" "github.com/matrix-org/dendrite/roomserver/acls"
"github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/api"
@ -35,10 +40,6 @@ import (
"github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/jetstream"
"github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/setup/process"
"github.com/matrix-org/gomatrixserverlib"
"github.com/nats-io/nats.go"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
) )
// Inputer is responsible for consuming from the roomserver input // Inputer is responsible for consuming from the roomserver input

10
roomserver/internal/input/input_latest_events.go
View file

@ -20,15 +20,16 @@ import (
"context" "context"
"fmt" "fmt"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/opentracing/opentracing-go"
"github.com/sirupsen/logrus"
"github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/state"
"github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/shared"
"github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/roomserver/types"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/opentracing/opentracing-go"
"github.com/sirupsen/logrus"
) )
// updateLatestEvents updates the list of latest events for this room in the database and writes the // updateLatestEvents updates the list of latest events for this room in the database and writes the
@ -36,7 +37,6 @@ import (
// The latest events are the events that aren't referenced by another event in the database: // The latest events are the events that aren't referenced by another event in the database:
// //
// Time goes down the page. 1 is the m.room.create event (root). // Time goes down the page. 1 is the m.room.create event (root).
//
// 1 After storing 1 the latest events are {1} // 1 After storing 1 the latest events are {1}
// | After storing 2 the latest events are {2} // | After storing 2 the latest events are {2}
// 2 After storing 3 the latest events are {3} // 2 After storing 3 the latest events are {3}

20
roomserver/internal/input/input_missing.go
View file

@ -326,8 +326,10 @@ func (t *missingStateReq) lookupStateAfterEvent(ctx context.Context, roomVersion
return respState, true, nil return respState, true, nil
} }
logrus.WithContext(ctx).Warnf("State for event %s not available locally, falling back to federation (via %d servers)", eventID, len(t.servers))
respState, err := t.lookupStateBeforeEvent(ctx, roomVersion, roomID, eventID) respState, err := t.lookupStateBeforeEvent(ctx, roomVersion, roomID, eventID)
if err != nil { if err != nil {
logrus.WithContext(ctx).WithError(err).Errorf("Failed to look up state before event %s", eventID)
return nil, false, fmt.Errorf("t.lookupStateBeforeEvent: %w", err) return nil, false, fmt.Errorf("t.lookupStateBeforeEvent: %w", err)
} }
@ -339,6 +341,7 @@ func (t *missingStateReq) lookupStateAfterEvent(ctx context.Context, roomVersion
case nil: case nil:
// do nothing // do nothing
default: default:
logrus.WithContext(ctx).WithError(err).Errorf("Failed to look up event %s", eventID)
return nil, false, fmt.Errorf("t.lookupEvent: %w", err) return nil, false, fmt.Errorf("t.lookupEvent: %w", err)
} }
h = t.cacheAndReturn(h) h = t.cacheAndReturn(h)
@ -662,9 +665,22 @@ func (t *missingStateReq) lookupMissingStateViaStateIDs(ctx context.Context, roo
util.GetLogger(ctx).WithField("room_id", roomID).Infof("lookupMissingStateViaStateIDs %s", eventID) util.GetLogger(ctx).WithField("room_id", roomID).Infof("lookupMissingStateViaStateIDs %s", eventID)
// fetch the state event IDs at the time of the event // fetch the state event IDs at the time of the event
stateIDs, err := t.federation.LookupStateIDs(ctx, t.origin, roomID, eventID) var stateIDs gomatrixserverlib.RespStateIDs
var err error
count := 0
totalctx, totalcancel := context.WithTimeout(ctx, time.Minute*5)
for _, serverName := range t.servers {
reqctx, reqcancel := context.WithTimeout(totalctx, time.Second*20)
stateIDs, err = t.federation.LookupStateIDs(reqctx, serverName, roomID, eventID)
reqcancel()
if err == nil {
break
}
count++
}
totalcancel()
if err != nil { if err != nil {
return nil, err return nil, fmt.Errorf("t.federation.LookupStateIDs tried %d server(s), last error: %w", count, err)
} }
// work out which auth/state IDs are missing // work out which auth/state IDs are missing
wantIDs := append(stateIDs.StateEventIDs, stateIDs.AuthEventIDs...) wantIDs := append(stateIDs.StateEventIDs, stateIDs.AuthEventIDs...)

8
roomserver/internal/perform/perform_backfill.go
View file

@ -19,6 +19,10 @@ import (
"fmt" "fmt"
"github.com/getsentry/sentry-go" "github.com/getsentry/sentry-go"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/sirupsen/logrus"
federationAPI "github.com/matrix-org/dendrite/federationapi/api" federationAPI "github.com/matrix-org/dendrite/federationapi/api"
"github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/api"
@ -26,9 +30,6 @@ import (
"github.com/matrix-org/dendrite/roomserver/internal/helpers" "github.com/matrix-org/dendrite/roomserver/internal/helpers"
"github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/storage"
"github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/roomserver/types"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/sirupsen/logrus"
) )
// the max number of servers to backfill from per request. If this is too low we may fail to backfill when // the max number of servers to backfill from per request. If this is too low we may fail to backfill when
@ -522,6 +523,7 @@ func (b *backfillRequester) ProvideEvents(roomVer gomatrixserverlib.RoomVersion,
} }
// joinEventsFromHistoryVisibility returns all CURRENTLY joined members if our server can read the room history // joinEventsFromHistoryVisibility returns all CURRENTLY joined members if our server can read the room history
//
// TODO: Long term we probably want a history_visibility table which stores eventNID | visibility_enum so we can just // TODO: Long term we probably want a history_visibility table which stores eventNID | visibility_enum so we can just
// pull all events and then filter by that table. // pull all events and then filter by that table.
func joinEventsFromHistoryVisibility( func joinEventsFromHistoryVisibility(

31
roomserver/storage/shared/storage.go
View file

@ -156,30 +156,15 @@ func (d *Database) RoomInfo(ctx context.Context, roomID string) (*types.RoomInfo
} }
func (d *Database) roomInfo(ctx context.Context, txn *sql.Tx, roomID string) (*types.RoomInfo, error) { func (d *Database) roomInfo(ctx context.Context, txn *sql.Tx, roomID string) (*types.RoomInfo, error) {
roomInfo, ok := d.Cache.GetRoomInfo(roomID) roomInfo, err := d.RoomsTable.SelectRoomInfo(ctx, txn, roomID)
if ok && roomInfo != nil && !roomInfo.IsStub() {
// The data that's in the cache is not stubby, so return it.
return roomInfo, nil
}
// At this point we either don't have an entry in the cache, or
// it is stubby, so let's check the roomserver_rooms table again.
roomInfoFromDB, err := d.RoomsTable.SelectRoomInfo(ctx, txn, roomID)
if err != nil { if err != nil {
return nil, err return nil, err
} }
// If we have a stubby cache entry already, update it and return
// the reference to the cache entry.
if roomInfo != nil { if roomInfo != nil {
roomInfo.CopyFrom(roomInfoFromDB) d.Cache.StoreRoomServerRoomID(roomInfo.RoomNID, roomID)
return roomInfo, nil d.Cache.StoreRoomVersion(roomID, roomInfo.RoomVersion)
} }
// Otherwise, try to admit the data into the cache and return the return roomInfo, err
// new reference from the database.
if roomInfoFromDB != nil {
d.Cache.StoreRoomServerRoomID(roomInfoFromDB.RoomNID, roomID)
d.Cache.StoreRoomInfo(roomID, roomInfoFromDB)
}
return roomInfoFromDB, err
} }
func (d *Database) AddState( func (d *Database) AddState(
@ -504,8 +489,8 @@ func (d *Database) events(
fetchNIDList := make([]types.RoomNID, 0, len(uniqueRoomNIDs)) fetchNIDList := make([]types.RoomNID, 0, len(uniqueRoomNIDs))
for n := range uniqueRoomNIDs { for n := range uniqueRoomNIDs {
if roomID, ok := d.Cache.GetRoomServerRoomID(n); ok { if roomID, ok := d.Cache.GetRoomServerRoomID(n); ok {
if roomInfo, ok := d.Cache.GetRoomInfo(roomID); ok { if roomVersion, ok := d.Cache.GetRoomVersion(roomID); ok {
roomVersions[n] = roomInfo.RoomVersion roomVersions[n] = roomVersion
continue continue
} }
} }
@ -762,9 +747,6 @@ func (d *Database) MissingAuthPrevEvents(
func (d *Database) assignRoomNID( func (d *Database) assignRoomNID(
ctx context.Context, roomID string, roomVersion gomatrixserverlib.RoomVersion, ctx context.Context, roomID string, roomVersion gomatrixserverlib.RoomVersion,
) (types.RoomNID, error) { ) (types.RoomNID, error) {
if roomInfo, ok := d.Cache.GetRoomInfo(roomID); ok {
return roomInfo.RoomNID, nil
}
// Check if we already have a numeric ID in the database. // Check if we already have a numeric ID in the database.
roomNID, err := d.RoomsTable.SelectRoomNID(ctx, nil, roomID) roomNID, err := d.RoomsTable.SelectRoomNID(ctx, nil, roomID)
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
@ -839,6 +821,7 @@ func extractRoomVersionFromCreateEvent(event *gomatrixserverlib.Event) (
// These cases are: // These cases are:
// - This is a redaction event, redact the event it references if we know about it. // - This is a redaction event, redact the event it references if we know about it.
// - This is a normal event which may have been previously redacted. // - This is a normal event which may have been previously redacted.
//
// In the first case, check if we have the referenced event then apply the redaction, else store it // In the first case, check if we have the referenced event then apply the redaction, else store it
// in the redactions table with validated=FALSE. In the second case, check if there is a redaction for it: // in the redactions table with validated=FALSE. In the second case, check if there is a redaction for it:
// if there is then apply the redactions and set validated=TRUE. // if there is then apply the redactions and set validated=TRUE.

3
roomserver/storage/tables/interface.go
View file

@ -5,9 +5,10 @@ import (
"database/sql" "database/sql"
"errors" "errors"
"github.com/matrix-org/dendrite/roomserver/types"
"github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib"
"github.com/tidwall/gjson" "github.com/tidwall/gjson"
"github.com/matrix-org/dendrite/roomserver/types"
) )
var OptimisationNotSupportedError = errors.New("optimisation not supported") var OptimisationNotSupportedError = errors.New("optimisation not supported")

2
run-sytest.sh
View file

@ -17,7 +17,7 @@ main() {
if [ -d ../sytest ]; then if [ -d ../sytest ]; then
local tmpdir local tmpdir
tmpdir="$(mktemp -d --tmpdir run-systest.XXXXXXXXXX)" tmpdir="$(mktemp -d -t run-systest.XXXXXXXXXX)"
trap "rm -r '$tmpdir'" EXIT trap "rm -r '$tmpdir'" EXIT
if [ -z "$DISABLE_BUILDING_SYTEST" ]; then if [ -z "$DISABLE_BUILDING_SYTEST" ]; then

14
setup/jetstream/nats.go
View file

@ -14,16 +14,16 @@ import (
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
natsserver "github.com/nats-io/nats-server/v2/server" natsserver "github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go"
natsclient "github.com/nats-io/nats.go" natsclient "github.com/nats-io/nats.go"
) )
type NATSInstance struct { type NATSInstance struct {
*natsserver.Server *natsserver.Server
sync.Mutex
} }
func DeleteAllStreams(js nats.JetStreamContext, cfg *config.JetStream) { var natsLock sync.Mutex
func DeleteAllStreams(js natsclient.JetStreamContext, cfg *config.JetStream) {
for _, stream := range streams { // streams are defined in streams.go for _, stream := range streams { // streams are defined in streams.go
name := cfg.Prefixed(stream.Name) name := cfg.Prefixed(stream.Name)
_ = js.DeleteStream(name) _ = js.DeleteStream(name)
@ -31,11 +31,12 @@ func DeleteAllStreams(js nats.JetStreamContext, cfg *config.JetStream) {
} }
func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetStream) (natsclient.JetStreamContext, *natsclient.Conn) { func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetStream) (natsclient.JetStreamContext, *natsclient.Conn) {
natsLock.Lock()
defer natsLock.Unlock()
// check if we need an in-process NATS Server // check if we need an in-process NATS Server
if len(cfg.Addresses) != 0 { if len(cfg.Addresses) != 0 {
return setupNATS(process, cfg, nil) return setupNATS(process, cfg, nil)
} }
s.Lock()
if s.Server == nil { if s.Server == nil {
var err error var err error
s.Server, err = natsserver.NewServer(&natsserver.Options{ s.Server, err = natsserver.NewServer(&natsserver.Options{
@ -63,7 +64,6 @@ func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetS
process.ComponentFinished() process.ComponentFinished()
}() }()
} }
s.Unlock()
if !s.ReadyForConnections(time.Second * 10) { if !s.ReadyForConnections(time.Second * 10) {
logrus.Fatalln("NATS did not start in time") logrus.Fatalln("NATS did not start in time")
} }
@ -77,9 +77,9 @@ func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetS
func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsclient.Conn) (natsclient.JetStreamContext, *natsclient.Conn) { func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsclient.Conn) (natsclient.JetStreamContext, *natsclient.Conn) {
if nc == nil { if nc == nil {
var err error var err error
opts := []nats.Option{} opts := []natsclient.Option{}
if cfg.DisableTLSValidation { if cfg.DisableTLSValidation {
opts = append(opts, nats.Secure(&tls.Config{ opts = append(opts, natsclient.Secure(&tls.Config{
InsecureSkipVerify: true, InsecureSkipVerify: true,
})) }))
} }

5
setup/mscs/msc2836/msc2836_test.go
View file

@ -15,6 +15,8 @@ import (
"time" "time"
"github.com/gorilla/mux" "github.com/gorilla/mux"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/dendrite/internal/hooks" "github.com/matrix-org/dendrite/internal/hooks"
"github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/httputil"
roomserver "github.com/matrix-org/dendrite/roomserver/api" roomserver "github.com/matrix-org/dendrite/roomserver/api"
@ -22,7 +24,6 @@ import (
"github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/mscs/msc2836" "github.com/matrix-org/dendrite/setup/mscs/msc2836"
userapi "github.com/matrix-org/dendrite/userapi/api" userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
) )
var ( var (
@ -32,6 +33,7 @@ var (
) )
// Basic sanity check of MSC2836 logic. Injects a thread that looks like: // Basic sanity check of MSC2836 logic. Injects a thread that looks like:
//
// A // A
// | // |
// B // B
@ -41,6 +43,7 @@ var (
// E F G // E F G
// | // |
// H // H
//
// And makes sure POST /event_relationships works with various parameters // And makes sure POST /event_relationships works with various parameters
func TestMSC2836(t *testing.T) { func TestMSC2836(t *testing.T) {
alice := "@alice:localhost" alice := "@alice:localhost"

52
syncapi/internal/keychange.go
View file

@ -25,10 +25,9 @@ import (
"github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/dendrite/syncapi/types"
"github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util" "github.com/matrix-org/util"
"github.com/sirupsen/logrus"
) )
const DeviceListLogName = "dl"
// DeviceOTKCounts adds one-time key counts to the /sync response // DeviceOTKCounts adds one-time key counts to the /sync response
func DeviceOTKCounts(ctx context.Context, keyAPI keyapi.SyncKeyAPI, userID, deviceID string, res *types.Response) error { func DeviceOTKCounts(ctx context.Context, keyAPI keyapi.SyncKeyAPI, userID, deviceID string, res *types.Response) error {
var queryRes keyapi.QueryOneTimeKeysResponse var queryRes keyapi.QueryOneTimeKeysResponse
@ -93,18 +92,13 @@ func DeviceListCatchup(
queryRes.UserIDs = append(queryRes.UserIDs, joinUserIDs...) queryRes.UserIDs = append(queryRes.UserIDs, joinUserIDs...)
queryRes.UserIDs = append(queryRes.UserIDs, leaveUserIDs...) queryRes.UserIDs = append(queryRes.UserIDs, leaveUserIDs...)
queryRes.UserIDs = util.UniqueStrings(queryRes.UserIDs) queryRes.UserIDs = util.UniqueStrings(queryRes.UserIDs)
var sharedUsersMap map[string]int sharedUsersMap := filterSharedUsers(ctx, db, userID, queryRes.UserIDs)
sharedUsersMap, queryRes.UserIDs = filterSharedUsers(ctx, db, userID, queryRes.UserIDs)
util.GetLogger(ctx).Debugf(
"QueryKeyChanges request off=%d,to=%d response off=%d uids=%v",
offset, toOffset, queryRes.Offset, queryRes.UserIDs,
)
userSet := make(map[string]bool) userSet := make(map[string]bool)
for _, userID := range res.DeviceLists.Changed { for _, userID := range res.DeviceLists.Changed {
userSet[userID] = true userSet[userID] = true
} }
for _, userID := range queryRes.UserIDs { for userID, count := range sharedUsersMap {
if !userSet[userID] { if !userSet[userID] && count > 0 {
res.DeviceLists.Changed = append(res.DeviceLists.Changed, userID) res.DeviceLists.Changed = append(res.DeviceLists.Changed, userID)
hasNew = true hasNew = true
userSet[userID] = true userSet[userID] = true
@ -113,7 +107,7 @@ func DeviceListCatchup(
// Finally, add in users who have joined or left. // Finally, add in users who have joined or left.
// TODO: This is sub-optimal because we will add users to `changed` even if we already shared a room with them. // TODO: This is sub-optimal because we will add users to `changed` even if we already shared a room with them.
for _, userID := range joinUserIDs { for _, userID := range joinUserIDs {
if !userSet[userID] { if !userSet[userID] && sharedUsersMap[userID] > 0 {
res.DeviceLists.Changed = append(res.DeviceLists.Changed, userID) res.DeviceLists.Changed = append(res.DeviceLists.Changed, userID)
hasNew = true hasNew = true
userSet[userID] = true userSet[userID] = true
@ -126,6 +120,13 @@ func DeviceListCatchup(
} }
} }
util.GetLogger(ctx).WithFields(logrus.Fields{
"user_id": userID,
"from": offset,
"to": toOffset,
"response_offset": queryRes.Offset,
}).Debugf("QueryKeyChanges request result: %+v", res.DeviceLists)
return types.StreamPosition(queryRes.Offset), hasNew, nil return types.StreamPosition(queryRes.Offset), hasNew, nil
} }
@ -220,24 +221,27 @@ func TrackChangedUsers(
// it down to include only users who the requesting user shares a room with. // it down to include only users who the requesting user shares a room with.
func filterSharedUsers( func filterSharedUsers(
ctx context.Context, db storage.SharedUsers, userID string, usersWithChangedKeys []string, ctx context.Context, db storage.SharedUsers, userID string, usersWithChangedKeys []string,
) (map[string]int, []string) { ) map[string]int {
sharedUsersMap := make(map[string]int, len(usersWithChangedKeys)) sharedUsersMap := make(map[string]int, len(usersWithChangedKeys))
for _, userID := range usersWithChangedKeys { for _, changedUserID := range usersWithChangedKeys {
sharedUsersMap[userID] = 0 sharedUsersMap[changedUserID] = 0
} if changedUserID == userID {
sharedUsers, err := db.SharedUsers(ctx, userID, usersWithChangedKeys)
if err != nil {
// default to all users so we do needless queries rather than miss some important device update
return nil, usersWithChangedKeys
}
for _, userID := range sharedUsers {
sharedUsersMap[userID]++
}
// We forcibly put ourselves in this list because we should be notified about our own device updates // We forcibly put ourselves in this list because we should be notified about our own device updates
// and if we are in 0 rooms then we don't technically share any room with ourselves so we wouldn't // and if we are in 0 rooms then we don't technically share any room with ourselves so we wouldn't
// be notified about key changes. // be notified about key changes.
sharedUsersMap[userID] = 1 sharedUsersMap[userID] = 1
return sharedUsersMap, sharedUsers }
}
sharedUsers, err := db.SharedUsers(ctx, userID, usersWithChangedKeys)
if err != nil {
util.GetLogger(ctx).WithError(err).Errorf("db.SharedUsers failed: %s", err)
// default to all users so we do needless queries rather than miss some important device update
return sharedUsersMap
}
for _, userID := range sharedUsers {
sharedUsersMap[userID]++
}
return sharedUsersMap
} }
func joinedRooms(res *types.Response, userID string) []string { func joinedRooms(res *types.Response, userID string) []string {

7
syncapi/internal/keychange_test.go
View file

@ -6,12 +6,13 @@ import (
"sort" "sort"
"testing" "testing"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
keyapi "github.com/matrix-org/dendrite/keyserver/api" keyapi "github.com/matrix-org/dendrite/keyserver/api"
"github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/dendrite/syncapi/types"
userapi "github.com/matrix-org/dendrite/userapi/api" userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
) )
var ( var (
@ -129,6 +130,7 @@ type wantCatchup struct {
} }
func assertCatchup(t *testing.T, hasNew bool, syncResponse *types.Response, want wantCatchup) { func assertCatchup(t *testing.T, hasNew bool, syncResponse *types.Response, want wantCatchup) {
t.Helper()
if hasNew != want.hasNew { if hasNew != want.hasNew {
t.Errorf("got hasNew=%v want %v", hasNew, want.hasNew) t.Errorf("got hasNew=%v want %v", hasNew, want.hasNew)
} }
@ -370,6 +372,7 @@ func TestKeyChangeCatchupChangeAndLeft(t *testing.T) {
// which are only relevant when actively sending events I think? And if Alice does need the keys she knows // which are only relevant when actively sending events I think? And if Alice does need the keys she knows
// charlie's (user_id, device_id) so can just hit /keys/query - no need to keep updated about it because she // charlie's (user_id, device_id) so can just hit /keys/query - no need to keep updated about it because she
// doesn't share any rooms with him. // doesn't share any rooms with him.
//
// Ergo, we put them in `left` as it is simpler. // Ergo, we put them in `left` as it is simpler.
func TestKeyChangeCatchupChangeAndLeftSameRoom(t *testing.T) { func TestKeyChangeCatchupChangeAndLeftSameRoom(t *testing.T) {
newShareUser := "@berta:localhost" newShareUser := "@berta:localhost"

11
syncapi/routing/filter.go
View file

@ -19,13 +19,14 @@ import (
"io/ioutil" "io/ioutil"
"net/http" "net/http"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/tidwall/gjson"
"github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/storage"
"github.com/matrix-org/dendrite/syncapi/sync" "github.com/matrix-org/dendrite/syncapi/sync"
"github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/tidwall/gjson"
) )
// GetFilter implements GET /_matrix/client/r0/user/{userId}/filter/{filterId} // GetFilter implements GET /_matrix/client/r0/user/{userId}/filter/{filterId}
@ -65,7 +66,9 @@ type filterResponse struct {
FilterID string `json:"filter_id"` FilterID string `json:"filter_id"`
} }
//PutFilter implements POST /_matrix/client/r0/user/{userId}/filter // PutFilter implements
//
// POST /_matrix/client/r0/user/{userId}/filter
func PutFilter( func PutFilter(
req *http.Request, device *api.Device, syncDB storage.Database, userID string, req *http.Request, device *api.Device, syncDB storage.Database, userID string,
) util.JSONResponse { ) util.JSONResponse {

4
syncapi/storage/postgres/current_room_state_table.go
View file

@ -112,7 +112,7 @@ const selectEventsWithEventIDsSQL = "" +
const selectSharedUsersSQL = "" + const selectSharedUsersSQL = "" +
"SELECT state_key FROM syncapi_current_room_state WHERE room_id = ANY(" + "SELECT state_key FROM syncapi_current_room_state WHERE room_id = ANY(" +
" SELECT room_id FROM syncapi_current_room_state WHERE state_key = $1 AND membership='join'" + " SELECT room_id FROM syncapi_current_room_state WHERE state_key = $1 AND membership='join'" +
") AND state_key = ANY($2) AND membership='join';" ") AND state_key = ANY($2) AND membership IN ('join', 'invite');"
type currentRoomStateStatements struct { type currentRoomStateStatements struct {
upsertRoomStateStmt *sql.Stmt upsertRoomStateStmt *sql.Stmt
@ -407,7 +407,7 @@ func (s *currentRoomStateStatements) SelectSharedUsers(
ctx context.Context, txn *sql.Tx, userID string, otherUserIDs []string, ctx context.Context, txn *sql.Tx, userID string, otherUserIDs []string,
) ([]string, error) { ) ([]string, error) {
stmt := sqlutil.TxStmt(txn, s.selectSharedUsersStmt) stmt := sqlutil.TxStmt(txn, s.selectSharedUsersStmt)
rows, err := stmt.QueryContext(ctx, userID, otherUserIDs) rows, err := stmt.QueryContext(ctx, userID, pq.Array(otherUserIDs))
if err != nil { if err != nil {
return nil, err return nil, err
} }

35
syncapi/storage/sqlite3/current_room_state_table.go
View file

@ -94,9 +94,9 @@ const selectEventsWithEventIDsSQL = "" +
" FROM syncapi_current_room_state WHERE event_id IN ($1)" " FROM syncapi_current_room_state WHERE event_id IN ($1)"
const selectSharedUsersSQL = "" + const selectSharedUsersSQL = "" +
"SELECT state_key FROM syncapi_current_room_state WHERE room_id = ANY(" + "SELECT state_key FROM syncapi_current_room_state WHERE room_id IN(" +
" SELECT room_id FROM syncapi_current_room_state WHERE state_key = $1 AND membership='join'" + " SELECT room_id FROM syncapi_current_room_state WHERE state_key = $1 AND membership='join'" +
") AND state_key IN ($2) AND membership='join';" ") AND state_key IN ($2) AND membership IN ('join', 'invite');"
type currentRoomStateStatements struct { type currentRoomStateStatements struct {
db *sql.DB db *sql.DB
@ -420,25 +420,28 @@ func (s *currentRoomStateStatements) SelectStateEvent(
func (s *currentRoomStateStatements) SelectSharedUsers( func (s *currentRoomStateStatements) SelectSharedUsers(
ctx context.Context, txn *sql.Tx, userID string, otherUserIDs []string, ctx context.Context, txn *sql.Tx, userID string, otherUserIDs []string,
) ([]string, error) { ) ([]string, error) {
query := strings.Replace(selectSharedUsersSQL, "($2)", sqlutil.QueryVariadicOffset(len(otherUserIDs), 1), 1)
stmt, err := s.db.Prepare(query)
if err != nil {
return nil, fmt.Errorf("SelectSharedUsers s.db.Prepare: %w", err)
}
defer internal.CloseAndLogIfError(ctx, stmt, "SelectSharedUsers: stmt.close() failed")
rows, err := sqlutil.TxStmt(txn, stmt).QueryContext(ctx, userID, otherUserIDs)
if err != nil {
return nil, err
}
defer internal.CloseAndLogIfError(ctx, rows, "selectSharedUsersStmt: rows.close() failed")
var stateKey string params := make([]interface{}, len(otherUserIDs)+1)
params[0] = userID
for k, v := range otherUserIDs {
params[k+1] = v
}
result := make([]string, 0, len(otherUserIDs)) result := make([]string, 0, len(otherUserIDs))
query := strings.Replace(selectSharedUsersSQL, "($2)", sqlutil.QueryVariadicOffset(len(otherUserIDs), 1), 1)
err := sqlutil.RunLimitedVariablesQuery(
ctx, query, s.db, params, sqlutil.SQLite3MaxVariables,
func(rows *sql.Rows) error {
var stateKey string
for rows.Next() { for rows.Next() {
if err := rows.Scan(&stateKey); err != nil { if err := rows.Scan(&stateKey); err != nil {
return nil, err return err
} }
result = append(result, stateKey) result = append(result, stateKey)
} }
return result, rows.Err() return nil
},
)
return result, err
} }

5
syncapi/storage/tables/interface.go
View file

@ -18,10 +18,11 @@ import (
"context" "context"
"database/sql" "database/sql"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/dendrite/syncapi/types"
"github.com/matrix-org/gomatrixserverlib"
) )
type AccountData interface { type AccountData interface {
@ -122,12 +123,14 @@ type CurrentRoomState interface {
// //
// We persist the previous event IDs as well, one per row, so when we do fetch even // We persist the previous event IDs as well, one per row, so when we do fetch even
// earlier events we can simply delete rows which referenced it. Consider the graph: // earlier events we can simply delete rows which referenced it. Consider the graph:
//
// A // A
// | Event C has 1 prev_event ID: A. // | Event C has 1 prev_event ID: A.
// B C // B C
// |___| Event D has 2 prev_event IDs: B and C. // |___| Event D has 2 prev_event IDs: B and C.
// | // |
// D // D
//
// The earliest known event we have is D, so this table has 2 rows. // The earliest known event we have is D, so this table has 2 rows.
// A backfill request gives us C but not B. We delete rows where prev_event=C. This // A backfill request gives us C but not B. We delete rows where prev_event=C. This
// still means that D is a backwards extremity as we do not have event B. However, event // still means that D is a backwards extremity as we do not have event B. However, event

7
syncapi/sync/requestpool_test.go
View file

@ -12,10 +12,13 @@ import (
) )
type dummyPublisher struct { type dummyPublisher struct {
lock sync.Mutex
count int count int
} }
func (d *dummyPublisher) SendPresence(userID string, presence types.Presence, statusMsg *string) error { func (d *dummyPublisher) SendPresence(userID string, presence types.Presence, statusMsg *string) error {
d.lock.Lock()
defer d.lock.Unlock()
d.count++ d.count++
return nil return nil
} }
@ -125,11 +128,15 @@ func TestRequestPool_updatePresence(t *testing.T) {
go rp.cleanPresence(db, time.Millisecond*50) go rp.cleanPresence(db, time.Millisecond*50)
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
publisher.lock.Lock()
beforeCount := publisher.count beforeCount := publisher.count
publisher.lock.Unlock()
rp.updatePresence(db, tt.args.presence, tt.args.userID) rp.updatePresence(db, tt.args.presence, tt.args.userID)
publisher.lock.Lock()
if tt.wantIncrease && publisher.count <= beforeCount { if tt.wantIncrease && publisher.count <= beforeCount {
t.Fatalf("expected count to increase: %d <= %d", publisher.count, beforeCount) t.Fatalf("expected count to increase: %d <= %d", publisher.count, beforeCount)
} }
publisher.lock.Unlock()
time.Sleep(tt.args.sleep) time.Sleep(tt.args.sleep)
}) })
} }

17
userapi/storage/sqlite3/stats_table.go
View file

@ -20,13 +20,14 @@ import (
"strings" "strings"
"time" "time"
"github.com/matrix-org/gomatrixserverlib"
"github.com/sirupsen/logrus"
"github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/dendrite/userapi/storage/tables" "github.com/matrix-org/dendrite/userapi/storage/tables"
"github.com/matrix-org/dendrite/userapi/types" "github.com/matrix-org/dendrite/userapi/types"
"github.com/matrix-org/gomatrixserverlib"
"github.com/sirupsen/logrus"
) )
const userDailyVisitsSchema = ` const userDailyVisitsSchema = `
@ -297,11 +298,10 @@ func (s *statsStatements) monthlyUsers(ctx context.Context, txn *sql.Tx) (result
return return
} }
/* R30Users counts the number of 30 day retained users, defined as: // R30Users counts the number of 30 day retained users, defined as:
- Users who have created their accounts more than 30 days ago // - Users who have created their accounts more than 30 days ago
- Where last seen at most 30 days ago // - Where last seen at most 30 days ago
- Where account creation and last_seen are > 30 days apart // - Where account creation and last_seen are > 30 days apart
*/
func (s *statsStatements) r30Users(ctx context.Context, txn *sql.Tx) (map[string]int64, error) { func (s *statsStatements) r30Users(ctx context.Context, txn *sql.Tx) (map[string]int64, error) {
stmt := sqlutil.TxStmt(txn, s.countR30UsersStmt) stmt := sqlutil.TxStmt(txn, s.countR30UsersStmt)
lastSeenAfter := time.Now().AddDate(0, 0, -30) lastSeenAfter := time.Now().AddDate(0, 0, -30)
@ -334,7 +334,8 @@ func (s *statsStatements) r30Users(ctx context.Context, txn *sql.Tx) (map[string
return result, rows.Err() return result, rows.Err()
} }
/* R30UsersV2 counts the number of 30 day retained users, defined as users that: /*
R30UsersV2 counts the number of 30 day retained users, defined as users that:
- Appear more than once in the past 60 days - Appear more than once in the past 60 days
- Have more than 30 days between the most and least recent appearances that occurred in the past 60 days. - Have more than 30 days between the most and least recent appearances that occurred in the past 60 days.
*/ */