mirrors/dendrite
1
0
Fork 0
mirror of https://github.com/matrix-org/dendrite.git synced 2026-01-05 21:23:10 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge commit '54f731c33f5828cafd6e84c438c358697118e7bd'

Browse source
This commit is contained in:
Brian Meek 2022-05-26 11:25:17 -07:00
parent da3d523037 16e261e3c2
commit 889c4cf9dc
108 changed files with 2258 additions and 2083 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
CHANGES.md
View file

@ -1,5 +1,39 @@
# Changelog
## Dendrite 0.8.6 (2022-05-26)
### Features
* Room versions 8 and 9 are now marked as stable
* Dendrite can now assist remote users to join restricted rooms via `/make_join` and `/send_join`
### Fixes
* The sync API no longer returns immediately on `/sync` requests unnecessarily if it can be avoided
* A race condition has been fixed in the sync API when updating presence via `/sync`
* A race condition has been fixed sending E2EE keys to remote servers over federation when joining rooms
* The `trusted_private_chat` preset should now grant power level 100 to all participant users, which should improve the user experience of direct messages
* Invited users are now authed correctly in restricted rooms
* The `join_authorised_by_users_server` key is now correctly stripped in restricted rooms when updating the membership event
* Appservices should now receive invite events correctly
* Device list updates should no longer contain optional fields with `null` values
* The `/deactivate` endpoint has been fixed to no longer confuse Element with incorrect completed flows
## Dendrite 0.8.5 (2022-05-13)
### Features
* New living documentation available at <https://matrix-org.github.io/dendrite/>, including new installation instructions
* The built-in NATS Server has been updated to version 2.8.2
### Fixes
* Monolith deployments will no longer panic at startup if given a config file that does not include the `internal_api` and `external_api` options
* State resolution v2 now correctly identifies other events related to power events, which should fix some event auth issues
* The latest events updater will no longer implicitly trust the new forward extremities when calculating the current room state, which may help to avoid some state resets
* The one-time key count is now correctly returned in `/sync` even if the request otherwise timed out, which should reduce the chance that unnecessary one-time keys will be uploaded by clients
* The `create-account` tool should now work properly when the database is configured using the global connection pool
## Dendrite 0.8.4 (2022-05-10)
### Fixes

2
README.md
View file

@ -53,7 +53,7 @@ The [Federation Tester](https://federationtester.matrix.org) can be used to veri
## Get started
If you wish to build a fully-federating Dendrite instance, see [the Installation documentation](docs/installation). For running in Docker, see [build/docker](build/docker).
If you wish to build a fully-federating Dendrite instance, see [the Installation documentation](https://matrix-org.github.io/dendrite/installation). For running in Docker, see [build/docker](build/docker).
The following instructions are enough to get Dendrite started as a non-federating test deployment using self-signed certificates and SQLite databases:

49
appservice/consumers/roomserver.go
View file

@ -83,29 +83,38 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msg *nats.Msg)
return true
}
if output.Type != api.OutputTypeNewRoomEvent || output.NewRoomEvent == nil {
return true
}
log.WithFields(log.Fields{
"type": output.Type,
}).Debug("Got a message in OutputRoomEventConsumer")
newEventID := output.NewRoomEvent.Event.EventID()
events := make([]*gomatrixserverlib.HeaderedEvent, 0, len(output.NewRoomEvent.AddsStateEventIDs))
events = append(events, output.NewRoomEvent.Event)
if len(output.NewRoomEvent.AddsStateEventIDs) > 0 {
eventsReq := &api.QueryEventsByIDRequest{
EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)),
}
eventsRes := &api.QueryEventsByIDResponse{}
for _, eventID := range output.NewRoomEvent.AddsStateEventIDs {
if eventID != newEventID {
eventsReq.EventIDs = append(eventsReq.EventIDs, eventID)
events := []*gomatrixserverlib.HeaderedEvent{}
if output.Type == api.OutputTypeNewRoomEvent && output.NewRoomEvent != nil {
newEventID := output.NewRoomEvent.Event.EventID()
events = append(events, output.NewRoomEvent.Event)
if len(output.NewRoomEvent.AddsStateEventIDs) > 0 {
eventsReq := &api.QueryEventsByIDRequest{
EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)),
}
eventsRes := &api.QueryEventsByIDResponse{}
for _, eventID := range output.NewRoomEvent.AddsStateEventIDs {
if eventID != newEventID {
eventsReq.EventIDs = append(eventsReq.EventIDs, eventID)
}
}
if len(eventsReq.EventIDs) > 0 {
if err := s.rsAPI.QueryEventsByID(s.ctx, eventsReq, eventsRes); err != nil {
return false
}
events = append(events, eventsRes.Events...)
}
}
if len(eventsReq.EventIDs) > 0 {
if err := s.rsAPI.QueryEventsByID(s.ctx, eventsReq, eventsRes); err != nil {
return false
}
events = append(events, eventsRes.Events...)
}
} else if output.Type == api.OutputTypeNewInviteEvent && output.NewInviteEvent != nil {
events = append(events, output.NewInviteEvent.Event)
} else {
log.WithFields(log.Fields{
"type": output.Type,
}).Debug("appservice OutputRoomEventConsumer ignoring event", string(msg.Data))
return true
}
// Send event to any relevant application services

2
build/gobind-pinecone/monolith.go
View file

@ -225,7 +225,7 @@ func (m *DendriteMonolith) Start() {
pk = sk.Public().(ed25519.PublicKey)
}
m.listener, err = net.Listen("tcp", ":65432")
m.listener, err = net.Listen("tcp", "localhost:65432")
if err != nil {
panic(err)
}

12
clientapi/auth/user_interactive.go
View file

@ -103,7 +103,8 @@ type userInteractiveFlow struct {
// that it isn't stolen by re-authenticating them.
type UserInteractive struct {
Completed []string
Flows []userInteractiveFlow
Flows []userInteractiveFlow
// Map of login type to implementation
Types map[string]Type
// Map of session ID to completed login types, will need to be extended in future
@ -129,6 +130,12 @@ func NewUserInteractive(
GetAccountByPassword: userAccountAPI.QueryAccountByPassword,
Config: cfg,
}
userInteractive.Flows = append(userInteractive.Flows, userInteractiveFlow{
Stages: []string{typePassword.Name()},
},
)
userInteractive.Types[typePassword.Name()] = typePassword
typePassword.AddFLows(&userInteractive)
}
@ -155,7 +162,6 @@ func (u *UserInteractive) IsSingleStageFlow(authType string) bool {
func (u *UserInteractive) AddCompletedStage(sessionID, authType string) {
// TODO: Handle multi-stage flows
u.Completed = append(u.Completed, authType)
delete(u.Sessions, sessionID)
}
@ -176,7 +182,7 @@ func (u *UserInteractive) Challenge(sessionID string) *util.JSONResponse {
return &util.JSONResponse{
Code: 401,
JSON: Challenge{
Completed: u.Completed,
Completed: u.Sessions[sessionID],
Flows: u.Flows,
Session: sessionID,
Params: u.Params,

35
clientapi/auth/user_interactive_test.go
View file

@ -186,3 +186,38 @@ func TestUserInteractivePasswordBadLogin(t *testing.T) {
}
}
}
func TestUserInteractive_AddCompletedStage(t *testing.T) {
tests := []struct {
name string
sessionID string
}{
{
name: "first user",
sessionID: util.RandomString(8),
},
{
name: "second user",
sessionID: util.RandomString(8),
},
{
name: "third user",
sessionID: util.RandomString(8),
},
}
u := setup()
ctx := context.Background()
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, resp := u.Verify(ctx, []byte("{}"))
challenge, ok := resp.JSON.(Challenge)
if !ok {
t.Fatalf("expected a Challenge, got %T", resp.JSON)
}
if len(challenge.Completed) > 0 {
t.Fatalf("expected 0 completed stages, got %d", len(challenge.Completed))
}
u.AddCompletedStage(tt.sessionID, "")
})
}
}

6
clientapi/jsonerror/jsonerror.go
View file

@ -154,6 +154,12 @@ func MissingParam(msg string) *MatrixError {
return &MatrixError{"M_MISSING_PARAM", msg}
}
// UnableToAuthoriseJoin is an error that is returned when a server can't
// determine whether to allow a restricted join or not.
func UnableToAuthoriseJoin(msg string) *MatrixError {
return &MatrixError{"M_UNABLE_TO_AUTHORISE_JOIN", msg}
}
// LeaveServerNoticeError is an error returned when trying to reject an invite
// for a server notice room.
func LeaveServerNoticeError() *MatrixError {

4
clientapi/routing/createroom.go
View file

@ -245,7 +245,9 @@ func createRoom(
case presetTrustedPrivateChat:
joinRuleContent.JoinRule = gomatrixserverlib.Invite
historyVisibilityContent.HistoryVisibility = historyVisibilityShared
// TODO If trusted_private_chat, all invitees are given the same power level as the room creator.
for _, invitee := range r.Invite {
powerLevelContent.Users[invitee] = 100
}
case presetPublicChat:
joinRuleContent.JoinRule = gomatrixserverlib.Public
historyVisibilityContent.HistoryVisibility = historyVisibilityShared

7
clientapi/routing/sendevent.go
View file

@ -104,6 +104,13 @@ func SendEvent(
return *resErr
}
// If we're sending a membership update, make sure to strip the authorised
// via key if it is present, otherwise other servers won't be able to auth
// the event if the room is set to the "restricted" join rule.
if eventType == gomatrixserverlib.MRoomMember {
delete(r, "join_authorised_via_users_server")
}
evTime, err := httputil.ParseTSParam(req)
if err != nil {
return util.JSONResponse{

2
cmd/generate-keys/main.go
View file

@ -20,7 +20,7 @@ import (
"log"
"os"
"github.com/matrix-org/dendrite/internal/test"
"github.com/matrix-org/dendrite/test"
)
const usage = `Usage: %s

22
cmd/resolve-state/main.go
View file

@ -4,13 +4,14 @@ import (
"context"
"flag"
"fmt"
"os"
"strconv"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/roomserver/storage"
"github.com/matrix-org/dendrite/roomserver/types"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
)
@ -23,11 +24,17 @@ import (
// e.g. ./resolve-state --roomversion=5 1254 1235 1282
var roomVersion = flag.String("roomversion", "5", "the room version to parse events as")
var filterType = flag.String("filtertype", "", "the event types to filter on")
func main() {
ctx := context.Background()
cfg := setup.ParseFlags(true)
args := os.Args[1:]
cfg.Logging = append(cfg.Logging[:0], config.LogrusHook{
Type: "std",
Level: "error",
})
base := base.NewBaseDendrite(cfg, "ResolveState", base.DisableMetrics)
args := flag.Args()
fmt.Println("Room version", *roomVersion)
@ -45,7 +52,7 @@ func main() {
panic(err)
}
roomserverDB, err := storage.Open(nil, &cfg.RoomServer.Database, cache)
roomserverDB, err := storage.Open(base, &cfg.RoomServer.Database, cache)
if err != nil {
panic(err)
}
@ -113,9 +120,18 @@ func main() {
}
fmt.Println("Resolved state contains", len(resolved), "events")
filteringEventType := *filterType
count := 0
for _, event := range resolved {
if filteringEventType != "" && event.Type() != filteringEventType {
continue
}
count++
fmt.Println()
fmt.Printf("* %s %s %q\n", event.EventID(), event.Type(), *event.StateKey())
fmt.Printf(" %s\n", string(event.Content()))
}
fmt.Println()
fmt.Println("Returned", count, "state events after filtering")
}

84
docs/coverage.md Normal file
View file

@ -0,0 +1,84 @@
---
title: Coverage
parent: Development
permalink: /development/coverage
---
To generate a test coverage report for Sytest, a small patch needs to be applied to the Sytest repository to compile and use the instrumented binary:
```patch
diff --git a/lib/SyTest/Homeserver/Dendrite.pm b/lib/SyTest/Homeserver/Dendrite.pm
index 8f0e209c..ad057e52 100644
--- a/lib/SyTest/Homeserver/Dendrite.pm
+++ b/lib/SyTest/Homeserver/Dendrite.pm
@@ -337,7 +337,7 @@ sub _start_monolith
$output->diag( "Starting monolith server" );
my @command = (
- $self->{bindir} . '/dendrite-monolith-server',
+ $self->{bindir} . '/dendrite-monolith-server', '--test.coverprofile=' . $self->{hs_dir} . '/integrationcover.log', "DEVEL",
'--config', $self->{paths}{config},
'--http-bind-address', $self->{bind_host} . ':' . $self->unsecure_port,
'--https-bind-address', $self->{bind_host} . ':' . $self->secure_port,
diff --git a/scripts/dendrite_sytest.sh b/scripts/dendrite_sytest.sh
index f009332b..7ea79869 100755
--- a/scripts/dendrite_sytest.sh
+++ b/scripts/dendrite_sytest.sh
@@ -34,7 +34,8 @@ export GOBIN=/tmp/bin
echo >&2 "--- Building dendrite from source"
cd /src
mkdir -p $GOBIN
-go install -v ./cmd/dendrite-monolith-server
+# go install -v ./cmd/dendrite-monolith-server
+go test -c -cover -covermode=atomic -o $GOBIN/dendrite-monolith-server -coverpkg "github.com/matrix-org/..." ./cmd/dendrite-monolith-server
go install -v ./cmd/generate-keys
cd -
```
Then run Sytest. This will generate a new file `integrationcover.log` in each server's directory e.g `server-0/integrationcover.log`. To parse it,
ensure your working directory is under the Dendrite repository then run:
```bash
go tool cover -func=/path/to/server-0/integrationcover.log
```
which will produce an output like:
```
...
github.com/matrix-org/util/json.go:83: NewJSONRequestHandler 100.0%
github.com/matrix-org/util/json.go:90: Protect 57.1%
github.com/matrix-org/util/json.go:110: RequestWithLogging 100.0%
github.com/matrix-org/util/json.go:132: MakeJSONAPI 70.0%
github.com/matrix-org/util/json.go:151: respond 61.5%
github.com/matrix-org/util/json.go:180: WithCORSOptions 0.0%
github.com/matrix-org/util/json.go:191: SetCORSHeaders 100.0%
github.com/matrix-org/util/json.go:202: RandomString 100.0%
github.com/matrix-org/util/json.go:210: init 100.0%
github.com/matrix-org/util/unique.go:13: Unique 91.7%
github.com/matrix-org/util/unique.go:48: SortAndUnique 100.0%
github.com/matrix-org/util/unique.go:55: UniqueStrings 100.0%
total: (statements) 53.7%
```
The total coverage for this run is the last line at the bottom. However, this value is misleading because Dendrite can run in many different configurations,
which will never be tested in a single test run (e.g sqlite or postgres, monolith or polylith). To get a more accurate value, additional processing is required
to remove packages which will never be tested and extension MSCs:
```bash
# These commands are all similar but change which package paths are _removed_ from the output.
# For Postgres (monolith)
go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'inthttp|sqlite|setup/mscs|api_trace' > coverage.txt
# For Postgres (polylith)
go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'sqlite|setup/mscs|api_trace' > coverage.txt
# For SQLite (monolith)
go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'inthttp|postgres|setup/mscs|api_trace' > coverage.txt
# For SQLite (polylith)
go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'postgres|setup/mscs|api_trace' > coverage.txt
```
A total value can then be calculated using:
```bash
cat coverage.txt | awk -F '\t+' '{x = x + $3} END {print x/NR}'
```
We currently do not have a way to combine Sytest/Complement/Unit Tests into a single coverage report.

22
docs/installation/1_planning.md
View file

@ -9,21 +9,19 @@ permalink: /installation/planning
## Modes
Dendrite can be run in one of two configurations:
Dendrite consists of several components, each responsible for a different aspect of the Matrix protocol.
Users can run Dendrite in one of two modes which dictate how these components are executed and communicate.
* **Monolith mode**: All components run in the same process. In this mode,
it is possible to run an in-process NATS Server instead of running a standalone deployment.
This will usually be the preferred model for low-to-mid volume deployments, providing the best
balance between performance and resource usage.
* **Monolith mode** runs all components in a single process. Components communicate through an internal NATS
server with generally low overhead. This mode dramatically simplifies deployment complexity and offers the
best balance between performance and resource usage for low-to-mid volume deployments.
* **Polylith mode**: A cluster of individual components running in their own processes, dealing
with different aspects of the Matrix protocol. Components communicate with each other using
internal HTTP APIs and NATS Server. This will almost certainly be the preferred model for very
large deployments but scalability comes with a cost. API calls are expensive and therefore a
polylith deployment may end up using disproportionately more resources for a smaller number of
users compared to a monolith deployment.
* **Polylith mode** runs all components in isolated processes. Components communicate through an external NATS
server and HTTP APIs, which incur considerable overhead. While this mode allows for more granular control of
resources dedicated toward individual processes, given the additional communications overhead, it is only
necessary for very large deployments.
At present, we **recommend monolith mode deployments** in all cases.
Given our current state of development, **we recommend monolith mode** for all deployments.
## Databases

40
federationapi/api/api.go
View file

@ -12,12 +12,16 @@ import (
// FederationInternalAPI is used to query information from the federation sender.
type FederationInternalAPI interface {
FederationClient
gomatrixserverlib.FederatedStateClient
KeyserverFederationAPI
gomatrixserverlib.KeyDatabase
ClientFederationAPI
RoomserverFederationAPI
QueryServerKeys(ctx context.Context, request *QueryServerKeysRequest, response *QueryServerKeysResponse) error
LookupServerKeys(ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) ([]gomatrixserverlib.ServerKeys, error)
MSC2836EventRelationships(ctx context.Context, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error)
MSC2946Spaces(ctx context.Context, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res gomatrixserverlib.MSC2946SpacesResponse, err error)
// Broadcasts an EDU to all servers in rooms we are joined to. Used in the yggdrasil demos.
PerformBroadcastEDU(
@ -60,17 +64,43 @@ type RoomserverFederationAPI interface {
LookupMissingEvents(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
}
// FederationClient is a subset of gomatrixserverlib.FederationClient functions which the fedsender
// KeyserverFederationAPI is a subset of gomatrixserverlib.FederationClient functions which the keyserver
// implements as proxy calls, with built-in backoff/retries/etc. Errors returned from functions in
// this interface are of type FederationClientError
type FederationClient interface {
gomatrixserverlib.FederatedStateClient
type KeyserverFederationAPI interface {
GetUserDevices(ctx context.Context, s gomatrixserverlib.ServerName, userID string) (res gomatrixserverlib.RespUserDevices, err error)
ClaimKeys(ctx context.Context, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (res gomatrixserverlib.RespClaimKeys, err error)
QueryKeys(ctx context.Context, s gomatrixserverlib.ServerName, keys map[string][]string) (res gomatrixserverlib.RespQueryKeys, err error)
}
// an interface for gmsl.FederationClient - contains functions called by federationapi only.
type FederationClient interface {
gomatrixserverlib.KeyClient
SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error)
// Perform operations
LookupRoomAlias(ctx context.Context, s gomatrixserverlib.ServerName, roomAlias string) (res gomatrixserverlib.RespDirectory, err error)
Peek(ctx context.Context, s gomatrixserverlib.ServerName, roomID, peekID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespPeek, err error)
MakeJoin(ctx context.Context, s gomatrixserverlib.ServerName, roomID, userID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMakeJoin, err error)
SendJoin(ctx context.Context, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res gomatrixserverlib.RespSendJoin, err error)
MakeLeave(ctx context.Context, s gomatrixserverlib.ServerName, roomID, userID string) (res gomatrixserverlib.RespMakeLeave, err error)
SendLeave(ctx context.Context, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (err error)
SendInviteV2(ctx context.Context, s gomatrixserverlib.ServerName, request gomatrixserverlib.InviteV2Request) (res gomatrixserverlib.RespInviteV2, err error)
GetEvent(ctx context.Context, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error)
GetEventAuth(ctx context.Context, s gomatrixserverlib.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res gomatrixserverlib.RespEventAuth, err error)
GetUserDevices(ctx context.Context, s gomatrixserverlib.ServerName, userID string) (gomatrixserverlib.RespUserDevices, error)
ClaimKeys(ctx context.Context, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (gomatrixserverlib.RespClaimKeys, error)
QueryKeys(ctx context.Context, s gomatrixserverlib.ServerName, keys map[string][]string) (gomatrixserverlib.RespQueryKeys, error)
Backfill(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, limit int, eventIDs []string) (res gomatrixserverlib.Transaction, err error)
MSC2836EventRelationships(ctx context.Context, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error)
MSC2946Spaces(ctx context.Context, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res gomatrixserverlib.MSC2946SpacesResponse, err error)
LookupServerKeys(ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) ([]gomatrixserverlib.ServerKeys, error)
ExchangeThirdPartyInvite(ctx context.Context, s gomatrixserverlib.ServerName, builder gomatrixserverlib.EventBuilder) (err error)
LookupState(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, eventID string, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespState, err error)
LookupStateIDs(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, eventID string) (res gomatrixserverlib.RespStateIDs, err error)
LookupMissingEvents(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
}
// FederationClientError is returned from FederationClient methods in the event of a problem.

5
federationapi/consumers/keychange.go
View file

@ -39,7 +39,7 @@ type KeyChangeConsumer struct {
db storage.Database
queues *queue.OutgoingQueues
serverName gomatrixserverlib.ServerName
rsAPI roomserverAPI.RoomserverInternalAPI
rsAPI roomserverAPI.FederationRoomserverAPI
topic string
}
@ -50,7 +50,7 @@ func NewKeyChangeConsumer(
js nats.JetStreamContext,
queues *queue.OutgoingQueues,
store storage.Database,
rsAPI roomserverAPI.RoomserverInternalAPI,
rsAPI roomserverAPI.FederationRoomserverAPI,
) *KeyChangeConsumer {
return &KeyChangeConsumer{
ctx: process.Context(),
@ -120,6 +120,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool {
logger.WithError(err).Error("failed to calculate joined rooms for user")
return true
}
// send this key change to all servers who share rooms with this user.
destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true)
if err != nil {

27
federationapi/consumers/roomserver.go
View file

@ -36,7 +36,7 @@ import (
type OutputRoomEventConsumer struct {
ctx context.Context
cfg *config.FederationAPI
rsAPI api.RoomserverInternalAPI
rsAPI api.FederationRoomserverAPI
jetstream nats.JetStreamContext
durable string
db storage.Database
@ -51,7 +51,7 @@ func NewOutputRoomEventConsumer(
js nats.JetStreamContext,
queues *queue.OutgoingQueues,
store storage.Database,
rsAPI api.RoomserverInternalAPI,
rsAPI api.FederationRoomserverAPI,
) *OutputRoomEventConsumer {
return &OutputRoomEventConsumer{
ctx: process.Context(),
@ -89,15 +89,7 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msg *nats.Msg)
switch output.Type {
case api.OutputTypeNewRoomEvent:
ev := output.NewRoomEvent.Event
if output.NewRoomEvent.RewritesState {
if err := s.db.PurgeRoomState(s.ctx, ev.RoomID()); err != nil {
log.WithError(err).Errorf("roomserver output log: purge room state failure")
return false
}
}
if err := s.processMessage(*output.NewRoomEvent); err != nil {
if err := s.processMessage(*output.NewRoomEvent, output.NewRoomEvent.RewritesState); err != nil {
// panic rather than continue with an inconsistent database
log.WithFields(log.Fields{
"event_id": ev.EventID(),
@ -145,7 +137,7 @@ func (s *OutputRoomEventConsumer) processInboundPeek(orp api.OutputNewInboundPee
// processMessage updates the list of currently joined hosts in the room
// and then sends the event to the hosts that were joined before the event.
func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) error {
func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rewritesState bool) error {
addsStateEvents, missingEventIDs := ore.NeededStateEventIDs()
// Ask the roomserver and add in the rest of the results into the set.
@ -164,7 +156,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) err
addsStateEvents = append(addsStateEvents, eventsRes.Events...)
}
addsJoinedHosts, err := joinedHostsFromEvents(gomatrixserverlib.UnwrapEventHeaders(addsStateEvents))
addsJoinedHosts, err := JoinedHostsFromEvents(gomatrixserverlib.UnwrapEventHeaders(addsStateEvents))
if err != nil {
return err
}
@ -176,10 +168,9 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) err
oldJoinedHosts, err := s.db.UpdateRoom(
s.ctx,
ore.Event.RoomID(),
ore.LastSentEventID,
ore.Event.EventID(),
addsJoinedHosts,
ore.RemovesStateEventIDs,
rewritesState, // if we're re-writing state, nuke all joined hosts before adding
)
if err != nil {
return err
@ -238,7 +229,7 @@ func (s *OutputRoomEventConsumer) joinedHostsAtEvent(
return nil, err
}
combinedAddsJoinedHosts, err := joinedHostsFromEvents(combinedAddsEvents)
combinedAddsJoinedHosts, err := JoinedHostsFromEvents(combinedAddsEvents)
if err != nil {
return nil, err
}
@ -284,10 +275,10 @@ func (s *OutputRoomEventConsumer) joinedHostsAtEvent(
return result, nil
}
// joinedHostsFromEvents turns a list of state events into a list of joined hosts.
// JoinedHostsFromEvents turns a list of state events into a list of joined hosts.
// This errors if one of the events was invalid.
// It should be impossible for an invalid event to get this far in the pipeline.
func joinedHostsFromEvents(evs []*gomatrixserverlib.Event) ([]types.JoinedHost, error) {
func JoinedHostsFromEvents(evs []*gomatrixserverlib.Event) ([]types.JoinedHost, error) {
var joinedHosts []types.JoinedHost
for _, ev := range evs {
if ev.Type() != "m.room.member" || ev.StateKey() == nil {

4
federationapi/federationapi.go
View file

@ -93,8 +93,8 @@ func AddPublicRoutes(
// can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
func NewInternalAPI(
base *base.BaseDendrite,
federation *gomatrixserverlib.FederationClient,
rsAPI roomserverAPI.RoomserverInternalAPI,
federation api.FederationClient,
rsAPI roomserverAPI.FederationRoomserverAPI,
caches *caching.Caches,
keyRing *gomatrixserverlib.KeyRing,
resetBlacklist bool,

106
federationapi/federationapi_keys_test.go
View file

@ -55,60 +55,62 @@ var servers = map[string]*server{
func TestMain(m *testing.M) {
// Set up the server key API for each "server" that we
// will use in our tests.
for _, s := range servers {
// Generate a new key.
_, testPriv, err := ed25519.GenerateKey(nil)
if err != nil {
panic("can't generate identity key: " + err.Error())
os.Exit(func() int {
for _, s := range servers {
// Generate a new key.
_, testPriv, err := ed25519.GenerateKey(nil)
if err != nil {
panic("can't generate identity key: " + err.Error())
}
// Create a new cache but don't enable prometheus!
s.cache, err = caching.NewInMemoryLRUCache(false)
if err != nil {
panic("can't create cache: " + err.Error())
}
// Create a temporary directory for JetStream.
d, err := ioutil.TempDir("./", "jetstream*")
if err != nil {
panic(err)
}
defer os.RemoveAll(d)
// Draw up just enough Dendrite config for the server key
// API to work.
cfg := &config.Dendrite{}
cfg.Defaults(true)
cfg.Global.ServerName = gomatrixserverlib.ServerName(s.name)
cfg.Global.PrivateKey = testPriv
cfg.Global.JetStream.InMemory = true
cfg.Global.JetStream.TopicPrefix = string(s.name[:1])
cfg.Global.JetStream.StoragePath = config.Path(d)
cfg.Global.KeyID = serverKeyID
cfg.Global.KeyValidityPeriod = s.validity
cfg.FederationAPI.Database.ConnectionString = config.DataSource("file::memory:")
s.config = &cfg.FederationAPI
// Create a transport which redirects federation requests to
// the mock round tripper. Since we're not *really* listening for
// federation requests then this will return the key instead.
transport := &http.Transport{}
transport.RegisterProtocol("matrix", &MockRoundTripper{})
// Create the federation client.
s.fedclient = gomatrixserverlib.NewFederationClient(
s.config.Matrix.ServerName, serverKeyID, testPriv,
gomatrixserverlib.WithTransport(transport),
)
// Finally, build the server key APIs.
sbase := base.NewBaseDendrite(cfg, "Monolith", base.DisableMetrics)
s.api = NewInternalAPI(sbase, s.fedclient, nil, s.cache, nil, true)
}
// Create a new cache but don't enable prometheus!
s.cache, err = caching.NewInMemoryLRUCache(false)
if err != nil {
panic("can't create cache: " + err.Error())
}
// Create a temporary directory for JetStream.
d, err := ioutil.TempDir("./", "jetstream*")
if err != nil {
panic(err)
}
defer os.RemoveAll(d)
// Draw up just enough Dendrite config for the server key
// API to work.
cfg := &config.Dendrite{}
cfg.Defaults(true)
cfg.Global.ServerName = gomatrixserverlib.ServerName(s.name)
cfg.Global.PrivateKey = testPriv
cfg.Global.JetStream.InMemory = true
cfg.Global.JetStream.TopicPrefix = string(s.name[:1])
cfg.Global.JetStream.StoragePath = config.Path(d)
cfg.Global.KeyID = serverKeyID
cfg.Global.KeyValidityPeriod = s.validity
cfg.FederationAPI.Database.ConnectionString = config.DataSource("file::memory:")
s.config = &cfg.FederationAPI
// Create a transport which redirects federation requests to
// the mock round tripper. Since we're not *really* listening for
// federation requests then this will return the key instead.
transport := &http.Transport{}
transport.RegisterProtocol("matrix", &MockRoundTripper{})
// Create the federation client.
s.fedclient = gomatrixserverlib.NewFederationClient(
s.config.Matrix.ServerName, serverKeyID, testPriv,
gomatrixserverlib.WithTransport(transport),
)
// Finally, build the server key APIs.
sbase := base.NewBaseDendrite(cfg, "Monolith", base.DisableMetrics)
s.api = NewInternalAPI(sbase, s.fedclient, nil, s.cache, nil, true)
}
// Now that we have built our server key APIs, start the
// rest of the tests.
os.Exit(m.Run())
// Now that we have built our server key APIs, start the
// rest of the tests.
return m.Run()
}())
}
type MockRoundTripper struct{}

236
federationapi/federationapi_test.go
View file

@ -3,18 +3,250 @@ package federationapi_test
import (
"context"
"crypto/ed25519"
"encoding/json"
"fmt"
"strings"
"testing"
"time"
"github.com/matrix-org/dendrite/federationapi"
"github.com/matrix-org/dendrite/federationapi/api"
"github.com/matrix-org/dendrite/federationapi/internal"
"github.com/matrix-org/dendrite/internal/test"
keyapi "github.com/matrix-org/dendrite/keyserver/api"
rsapi "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/jetstream"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/test/testrig"
"github.com/matrix-org/gomatrix"
"github.com/matrix-org/gomatrixserverlib"
"github.com/nats-io/nats.go"
)
type fedRoomserverAPI struct {
rsapi.FederationRoomserverAPI
inputRoomEvents func(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse)
queryRoomsForUser func(ctx context.Context, req *rsapi.QueryRoomsForUserRequest, res *rsapi.QueryRoomsForUserResponse) error
}
// PerformJoin will call this function
func (f *fedRoomserverAPI) InputRoomEvents(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse) {
if f.inputRoomEvents == nil {
return
}
f.inputRoomEvents(ctx, req, res)
}
// keychange consumer calls this
func (f *fedRoomserverAPI) QueryRoomsForUser(ctx context.Context, req *rsapi.QueryRoomsForUserRequest, res *rsapi.QueryRoomsForUserResponse) error {
if f.queryRoomsForUser == nil {
return nil
}
return f.queryRoomsForUser(ctx, req, res)
}
// TODO: This struct isn't generic, only works for TestFederationAPIJoinThenKeyUpdate
type fedClient struct {
api.FederationClient
allowJoins []*test.Room
keys map[gomatrixserverlib.ServerName]struct {
key ed25519.PrivateKey
keyID gomatrixserverlib.KeyID
}
t *testing.T
sentTxn bool
}
func (f *fedClient) GetServerKeys(ctx context.Context, matrixServer gomatrixserverlib.ServerName) (gomatrixserverlib.ServerKeys, error) {
fmt.Println("GetServerKeys:", matrixServer)
var keys gomatrixserverlib.ServerKeys
var keyID gomatrixserverlib.KeyID
var pkey ed25519.PrivateKey
for srv, data := range f.keys {
if srv == matrixServer {
pkey = data.key
keyID = data.keyID
break
}
}
if pkey == nil {
return keys, nil
}
keys.ServerName = matrixServer
keys.ValidUntilTS = gomatrixserverlib.AsTimestamp(time.Now().Add(10 * time.Hour))
publicKey := pkey.Public().(ed25519.PublicKey)
keys.VerifyKeys = map[gomatrixserverlib.KeyID]gomatrixserverlib.VerifyKey{
keyID: {
Key: gomatrixserverlib.Base64Bytes(publicKey),
},
}
toSign, err := json.Marshal(keys.ServerKeyFields)
if err != nil {
return keys, err
}
keys.Raw, err = gomatrixserverlib.SignJSON(
string(matrixServer), keyID, pkey, toSign,
)
if err != nil {
return keys, err
}
return keys, nil
}
func (f *fedClient) MakeJoin(ctx context.Context, s gomatrixserverlib.ServerName, roomID, userID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMakeJoin, err error) {
for _, r := range f.allowJoins {
if r.ID == roomID {
res.RoomVersion = r.Version
res.JoinEvent = gomatrixserverlib.EventBuilder{
Sender: userID,
RoomID: roomID,
Type: "m.room.member",
StateKey: &userID,
Content: gomatrixserverlib.RawJSON([]byte(`{"membership":"join"}`)),
PrevEvents: r.ForwardExtremities(),
}
var needed gomatrixserverlib.StateNeeded
needed, err = gomatrixserverlib.StateNeededForEventBuilder(&res.JoinEvent)
if err != nil {
f.t.Errorf("StateNeededForEventBuilder: %v", err)
return
}
res.JoinEvent.AuthEvents = r.MustGetAuthEventRefsForEvent(f.t, needed)
return
}
}
return
}
func (f *fedClient) SendJoin(ctx context.Context, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res gomatrixserverlib.RespSendJoin, err error) {
for _, r := range f.allowJoins {
if r.ID == event.RoomID() {
r.InsertEvent(f.t, event.Headered(r.Version))
f.t.Logf("Join event: %v", event.EventID())
res.StateEvents = gomatrixserverlib.NewEventJSONsFromHeaderedEvents(r.CurrentState())
res.AuthEvents = gomatrixserverlib.NewEventJSONsFromHeaderedEvents(r.Events())
}
}
return
}
func (f *fedClient) SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error) {
for _, edu := range t.EDUs {
if edu.Type == gomatrixserverlib.MDeviceListUpdate {
f.sentTxn = true
}
}
f.t.Logf("got /send")
return
}
// Regression test to make sure that /send_join is updating the destination hosts synchronously and
// isn't relying on the roomserver.
func TestFederationAPIJoinThenKeyUpdate(t *testing.T) {
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
testFederationAPIJoinThenKeyUpdate(t, dbType)
})
}
func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) {
base, close := testrig.CreateBaseDendrite(t, dbType)
base.Cfg.FederationAPI.PreferDirectFetch = true
defer close()
jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream)
serverA := gomatrixserverlib.ServerName("server.a")
serverAKeyID := gomatrixserverlib.KeyID("ed25519:servera")
serverAPrivKey := test.PrivateKeyA
creator := test.NewUser(t, test.WithSigningServer(serverA, serverAKeyID, serverAPrivKey))
myServer := base.Cfg.Global.ServerName
myServerKeyID := base.Cfg.Global.KeyID
myServerPrivKey := base.Cfg.Global.PrivateKey
joiningUser := test.NewUser(t, test.WithSigningServer(myServer, myServerKeyID, myServerPrivKey))
fmt.Printf("creator: %v joining user: %v\n", creator.ID, joiningUser.ID)
room := test.NewRoom(t, creator)
rsapi := &fedRoomserverAPI{
inputRoomEvents: func(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse) {
if req.Asynchronous {
t.Errorf("InputRoomEvents from PerformJoin MUST be synchronous")
}
},
queryRoomsForUser: func(ctx context.Context, req *rsapi.QueryRoomsForUserRequest, res *rsapi.QueryRoomsForUserResponse) error {
if req.UserID == joiningUser.ID && req.WantMembership == "join" {
res.RoomIDs = []string{room.ID}
return nil
}
return fmt.Errorf("unexpected queryRoomsForUser: %+v", *req)
},
}
fc := &fedClient{
allowJoins: []*test.Room{room},
t: t,
keys: map[gomatrixserverlib.ServerName]struct {
key ed25519.PrivateKey
keyID gomatrixserverlib.KeyID
}{
serverA: {
key: serverAPrivKey,
keyID: serverAKeyID,
},
myServer: {
key: myServerPrivKey,
keyID: myServerKeyID,
},
},
}
fsapi := federationapi.NewInternalAPI(base, fc, rsapi, base.Caches, nil, false)
var resp api.PerformJoinResponse
fsapi.PerformJoin(context.Background(), &api.PerformJoinRequest{
RoomID: room.ID,
UserID: joiningUser.ID,
ServerNames: []gomatrixserverlib.ServerName{serverA},
}, &resp)
if resp.JoinedVia != serverA {
t.Errorf("PerformJoin: joined via %v want %v", resp.JoinedVia, serverA)
}
if resp.LastError != nil {
t.Fatalf("PerformJoin: returned error: %+v", *resp.LastError)
}
// Inject a keyserver key change event and ensure we try to send it out. If we don't, then the
// federationapi is incorrectly waiting for an output room event to arrive to update the joined
// hosts table.
key := keyapi.DeviceMessage{
Type: keyapi.TypeDeviceKeyUpdate,
DeviceKeys: &keyapi.DeviceKeys{
UserID: joiningUser.ID,
DeviceID: "MY_DEVICE",
DisplayName: "BLARGLE",
KeyJSON: []byte(`{}`),
},
}
b, err := json.Marshal(key)
if err != nil {
t.Fatalf("Failed to marshal device message: %s", err)
}
msg := &nats.Msg{
Subject: base.Cfg.Global.JetStream.Prefixed(jetstream.OutputKeyChangeEvent),
Header: nats.Header{},
Data: b,
}
msg.Header.Set(jetstream.UserID, key.UserID)
testrig.MustPublishMsgs(t, jsctx, msg)
time.Sleep(500 * time.Millisecond)
if !fc.sentTxn {
t.Fatalf("did not send device list update")
}
}
// Tests that event IDs with '/' in them (escaped as %2F) are correctly passed to the right handler and don't 404.
// Relevant for v3 rooms and a cause of flakey sytests as the IDs are randomly generated.
func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
@ -86,7 +318,7 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
}
gerr, ok := err.(gomatrix.HTTPError)
if !ok {
t.Errorf("failed to cast response error as gomatrix.HTTPError")
t.Errorf("failed to cast response error as gomatrix.HTTPError: %s", err)
continue
}
t.Logf("Error: %+v", gerr)

8
federationapi/internal/api.go
View file

@ -25,8 +25,8 @@ type FederationInternalAPI struct {
db storage.Database
cfg *config.FederationAPI
statistics *statistics.Statistics
rsAPI roomserverAPI.RoomserverInternalAPI
federation *gomatrixserverlib.FederationClient
rsAPI roomserverAPI.FederationRoomserverAPI
federation api.FederationClient
keyRing *gomatrixserverlib.KeyRing
queues *queue.OutgoingQueues
joins sync.Map // joins currently in progress
@ -34,8 +34,8 @@ type FederationInternalAPI struct {
func NewFederationInternalAPI(
db storage.Database, cfg *config.FederationAPI,
rsAPI roomserverAPI.RoomserverInternalAPI,
federation *gomatrixserverlib.FederationClient,
rsAPI roomserverAPI.FederationRoomserverAPI,
federation api.FederationClient,
statistics *statistics.Statistics,
caches *caching.Caches,
queues *queue.OutgoingQueues,

55
federationapi/internal/perform.go
View file

@ -8,6 +8,7 @@ import (
"time"
"github.com/matrix-org/dendrite/federationapi/api"
"github.com/matrix-org/dendrite/federationapi/consumers"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/roomserver/version"
"github.com/matrix-org/gomatrix"
@ -165,7 +166,8 @@ func (r *FederationInternalAPI) performJoinUsingServer(
if content == nil {
content = map[string]interface{}{}
}
content["membership"] = "join"
_ = json.Unmarshal(respMakeJoin.JoinEvent.Content, &content)
content["membership"] = gomatrixserverlib.Join
if err = respMakeJoin.JoinEvent.SetContent(content); err != nil {
return fmt.Errorf("respMakeJoin.JoinEvent.SetContent: %w", err)
}
@ -208,10 +210,22 @@ func (r *FederationInternalAPI) performJoinUsingServer(
}
r.statistics.ForServer(serverName).Success()
authEvents := respSendJoin.AuthEvents.UntrustedEvents(respMakeJoin.RoomVersion)
// If the remote server returned an event in the "event" key of
// the send_join request then we should use that instead. It may
// contain signatures that we don't know about.
if len(respSendJoin.Event) > 0 {
var remoteEvent *gomatrixserverlib.Event
remoteEvent, err = respSendJoin.Event.UntrustedEvent(respMakeJoin.RoomVersion)
if err == nil && isWellFormedMembershipEvent(
remoteEvent, roomID, userID, r.cfg.Matrix.ServerName,
) {
event = remoteEvent
}
}
// Sanity-check the join response to ensure that it has a create
// event, that the room version is known, etc.
authEvents := respSendJoin.AuthEvents.UntrustedEvents(respMakeJoin.RoomVersion)
if err = sanityCheckAuthChain(authEvents); err != nil {
return fmt.Errorf("sanityCheckAuthChain: %w", err)
}
@ -235,6 +249,21 @@ func (r *FederationInternalAPI) performJoinUsingServer(
return fmt.Errorf("respSendJoin.Check: %w", err)
}
// We need to immediately update our list of joined hosts for this room now as we are technically
// joined. We must do this synchronously: we cannot rely on the roomserver output events as they
// will happen asyncly. If we don't update this table, you can end up with bad failure modes like
// joining a room, waiting for 200 OK then changing device keys and have those keys not be sent
// to other servers (this was a cause of a flakey sytest "Local device key changes get to remote servers")
// The events are trusted now as we performed auth checks above.
joinedHosts, err := consumers.JoinedHostsFromEvents(respState.StateEvents.TrustedEvents(respMakeJoin.RoomVersion, false))
if err != nil {
return fmt.Errorf("JoinedHostsFromEvents: failed to get joined hosts: %s", err)
}
logrus.WithField("hosts", joinedHosts).WithField("room", roomID).Info("Joined federated room with hosts")
if _, err = r.db.UpdateRoom(context.Background(), roomID, joinedHosts, nil, true); err != nil {
return fmt.Errorf("UpdatedRoom: failed to update room with joined hosts: %s", err)
}
// If we successfully performed a send_join above then the other
// server now thinks we're a part of the room. Send the newly
// returned state to the roomserver to update our local view.
@ -254,6 +283,26 @@ func (r *FederationInternalAPI) performJoinUsingServer(
return nil
}
// isWellFormedMembershipEvent returns true if the event looks like a legitimate
// membership event.
func isWellFormedMembershipEvent(event *gomatrixserverlib.Event, roomID, userID string, origin gomatrixserverlib.ServerName) bool {
if membership, err := event.Membership(); err != nil {
return false
} else if membership != gomatrixserverlib.Join {
return false
}
if event.RoomID() != roomID {
return false
}
if event.Origin() != origin {
return false
}
if !event.StateKeyEquals(userID) {
return false
}
return true
}
// PerformOutboundPeekRequest implements api.FederationInternalAPI
func (r *FederationInternalAPI) PerformOutboundPeek(
ctx context.Context,
@ -650,7 +699,7 @@ func setDefaultRoomVersionFromJoinEvent(joinEvent gomatrixserverlib.EventBuilder
// FederatedAuthProvider is an auth chain provider which fetches events from the server provided
func federatedAuthProvider(
ctx context.Context, federation *gomatrixserverlib.FederationClient,
ctx context.Context, federation api.FederationClient,
keyRing gomatrixserverlib.JSONVerifier, server gomatrixserverlib.ServerName,
) gomatrixserverlib.AuthChainProvider {
// A list of events that we have retried, if they were not included in

31
federationapi/queue/destinationqueue.go
View file

@ -21,6 +21,7 @@ import (
"sync"
"time"
fedapi "github.com/matrix-org/dendrite/federationapi/api"
"github.com/matrix-org/dendrite/federationapi/statistics"
"github.com/matrix-org/dendrite/federationapi/storage"
"github.com/matrix-org/dendrite/federationapi/storage/shared"
@ -49,21 +50,21 @@ type destinationQueue struct {
db storage.Database
process *process.ProcessContext
signing *SigningInfo
rsAPI api.RoomserverInternalAPI
client *gomatrixserverlib.FederationClient // federation client
origin gomatrixserverlib.ServerName // origin of requests
destination gomatrixserverlib.ServerName // destination of requests
running atomic.Bool // is the queue worker running?
backingOff atomic.Bool // true if we're backing off
overflowed atomic.Bool // the queues exceed maxPDUsInMemory/maxEDUsInMemory, so we should consult the database for more
statistics *statistics.ServerStatistics // statistics about this remote server
transactionIDMutex sync.Mutex // protects transactionID
transactionID gomatrixserverlib.TransactionID // last transaction ID if retrying, or "" if last txn was successful
notify chan struct{} // interrupts idle wait pending PDUs/EDUs
pendingPDUs []*queuedPDU // PDUs waiting to be sent
pendingEDUs []*queuedEDU // EDUs waiting to be sent
pendingMutex sync.RWMutex // protects pendingPDUs and pendingEDUs
interruptBackoff chan bool // interrupts backoff
rsAPI api.FederationRoomserverAPI
client fedapi.FederationClient // federation client
origin gomatrixserverlib.ServerName // origin of requests
destination gomatrixserverlib.ServerName // destination of requests
running atomic.Bool // is the queue worker running?
backingOff atomic.Bool // true if we're backing off
overflowed atomic.Bool // the queues exceed maxPDUsInMemory/maxEDUsInMemory, so we should consult the database for more
statistics *statistics.ServerStatistics // statistics about this remote server
transactionIDMutex sync.Mutex // protects transactionID
transactionID gomatrixserverlib.TransactionID // last transaction ID if retrying, or "" if last txn was successful
notify chan struct{} // interrupts idle wait pending PDUs/EDUs
pendingPDUs []*queuedPDU // PDUs waiting to be sent
pendingEDUs []*queuedEDU // EDUs waiting to be sent
pendingMutex sync.RWMutex // protects pendingPDUs and pendingEDUs
interruptBackoff chan bool // interrupts backoff
}
// Send event adds the event to the pending queue for the destination.

9
federationapi/queue/queue.go
View file

@ -26,6 +26,7 @@ import (
log "github.com/sirupsen/logrus"
"github.com/tidwall/gjson"
fedapi "github.com/matrix-org/dendrite/federationapi/api"
"github.com/matrix-org/dendrite/federationapi/statistics"
"github.com/matrix-org/dendrite/federationapi/storage"
"github.com/matrix-org/dendrite/federationapi/storage/shared"
@ -39,9 +40,9 @@ type OutgoingQueues struct {
db storage.Database
process *process.ProcessContext
disabled bool
rsAPI api.RoomserverInternalAPI
rsAPI api.FederationRoomserverAPI
origin gomatrixserverlib.ServerName
client *gomatrixserverlib.FederationClient
client fedapi.FederationClient
statistics *statistics.Statistics
signing *SigningInfo
queuesMutex sync.Mutex // protects the below
@ -85,8 +86,8 @@ func NewOutgoingQueues(
process *process.ProcessContext,
disabled bool,
origin gomatrixserverlib.ServerName,
client *gomatrixserverlib.FederationClient,
rsAPI api.RoomserverInternalAPI,
client fedapi.FederationClient,
rsAPI api.FederationRoomserverAPI,
statistics *statistics.Statistics,
signing *SigningInfo,
) *OutgoingQueues {

122
federationapi/routing/join.go
View file

@ -15,6 +15,7 @@
package routing
import (
"encoding/json"
"fmt"
"net/http"
"sort"
@ -103,6 +104,16 @@ func MakeJoin(
}
}
// Check if the restricted join is allowed. If the room doesn't
// support restricted joins then this is effectively a no-op.
res, authorisedVia, err := checkRestrictedJoin(httpReq, rsAPI, verRes.RoomVersion, roomID, userID)
if err != nil {
util.GetLogger(httpReq.Context()).WithError(err).Error("checkRestrictedJoin failed")
return jsonerror.InternalServerError()
} else if res != nil {
return *res
}
// Try building an event for the server
builder := gomatrixserverlib.EventBuilder{
Sender: userID,
@ -110,8 +121,11 @@ func MakeJoin(
Type: "m.room.member",
StateKey: &userID,
}
err = builder.SetContent(map[string]interface{}{"membership": gomatrixserverlib.Join})
if err != nil {
content := gomatrixserverlib.MemberContent{
Membership: gomatrixserverlib.Join,
AuthorisedVia: authorisedVia,
}
if err = builder.SetContent(content); err != nil {
util.GetLogger(httpReq.Context()).WithError(err).Error("builder.SetContent failed")
return jsonerror.InternalServerError()
}
@ -161,6 +175,7 @@ func MakeJoin(
// SendJoin implements the /send_join API
// The make-join send-join dance makes much more sense as a single
// flow so the cyclomatic complexity is high:
// nolint:gocyclo
func SendJoin(
httpReq *http.Request,
request *gomatrixserverlib.FederationRequest,
@ -314,6 +329,40 @@ func SendJoin(
}
}
// If the membership content contains a user ID for a server that is not
// ours then we should kick it back.
var memberContent gomatrixserverlib.MemberContent
if err := json.Unmarshal(event.Content(), &memberContent); err != nil {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.BadJSON(err.Error()),
}
}
if memberContent.AuthorisedVia != "" {
_, domain, err := gomatrixserverlib.SplitID('@', memberContent.AuthorisedVia)
if err != nil {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.BadJSON(fmt.Sprintf("The authorising username %q is invalid.", memberContent.AuthorisedVia)),
}
}
if domain != cfg.Matrix.ServerName {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.BadJSON(fmt.Sprintf("The authorising username %q does not belong to this server.", memberContent.AuthorisedVia)),
}
}
}
// Sign the membership event. This is required for restricted joins to work
// in the case that the authorised via user is one of our own users. It also
// doesn't hurt to do it even if it isn't a restricted join.
signed := event.Sign(
string(cfg.Matrix.ServerName),
cfg.Matrix.KeyID,
cfg.Matrix.PrivateKey,
)
// Send the events to the room server.
// We are responsible for notifying other servers that the user has joined
// the room, so set SendAsServer to cfg.Matrix.ServerName
@ -323,7 +372,7 @@ func SendJoin(
InputRoomEvents: []api.InputRoomEvent{
{
Kind: api.KindNew,
Event: event.Headered(stateAndAuthChainResponse.RoomVersion),
Event: signed.Headered(stateAndAuthChainResponse.RoomVersion),
SendAsServer: string(cfg.Matrix.ServerName),
TransactionID: nil,
},
@ -354,10 +403,77 @@ func SendJoin(
StateEvents: gomatrixserverlib.NewEventJSONsFromHeaderedEvents(stateAndAuthChainResponse.StateEvents),
AuthEvents: gomatrixserverlib.NewEventJSONsFromHeaderedEvents(stateAndAuthChainResponse.AuthChainEvents),
Origin: cfg.Matrix.ServerName,
Event: signed.JSON(),
},
}
}
// checkRestrictedJoin finds out whether or not we can assist in processing
// a restricted room join. If the room version does not support restricted
// joins then this function returns with no side effects. This returns three
// values:
// * an optional JSON response body (i.e. M_UNABLE_TO_AUTHORISE_JOIN) which
// should always be sent back to the client if one is specified
// * a user ID of an authorising user, typically a user that has power to
// issue invites in the room, if one has been found
// * an error if there was a problem finding out if this was allowable,
// like if the room version isn't known or a problem happened talking to
// the roomserver
func checkRestrictedJoin(
httpReq *http.Request,
rsAPI api.FederationRoomserverAPI,
roomVersion gomatrixserverlib.RoomVersion,
roomID, userID string,
) (*util.JSONResponse, string, error) {
if allowRestricted, err := roomVersion.MayAllowRestrictedJoinsInEventAuth(); err != nil {
return nil, "", err
} else if !allowRestricted {
return nil, "", nil
}
req := &api.QueryRestrictedJoinAllowedRequest{
RoomID: roomID,
UserID: userID,
}
res := &api.QueryRestrictedJoinAllowedResponse{}
if err := rsAPI.QueryRestrictedJoinAllowed(httpReq.Context(), req, res); err != nil {
return nil, "", err
}
switch {
case !res.Restricted:
// The join rules for the room don't restrict membership.
return nil, "", nil
case !res.Resident:
// The join rules restrict membership but our server isn't currently
// joined to all of the allowed rooms, so we can't actually decide
// whether or not to allow the user to join. This error code should
// tell the joining server to try joining via another resident server
// instead.
return &util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.UnableToAuthoriseJoin("This server cannot authorise the join."),
}, "", nil
case !res.Allowed:
// The join rules restrict membership, our server is in the relevant
// rooms and the user wasn't joined to join any of the allowed rooms
// and therefore can't join this room.
return &util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("You are not joined to any matching rooms."),
}, "", nil
default:
// The join rules restrict membership, our server is in the relevant
// rooms and the user was allowed to join because they belong to one
// of the allowed rooms. We now need to pick one of our own local users
// from within the room to use as the authorising user ID, so that it
// can be referred to from within the membership content.
return nil, res.AuthorisedVia, nil
}
}
type eventsByDepth []*gomatrixserverlib.HeaderedEvent
func (e eventsByDepth) Len() int {

2
federationapi/routing/query.go
View file

@ -30,7 +30,7 @@ import (
// RoomAliasToID converts the queried alias into a room ID and returns it
func RoomAliasToID(
httpReq *http.Request,
federation *gomatrixserverlib.FederationClient,
federation federationAPI.FederationClient,
cfg *config.FederationAPI,
rsAPI roomserverAPI.FederationRoomserverAPI,
senderAPI federationAPI.FederationInternalAPI,

2
federationapi/routing/routing.go
View file

@ -54,7 +54,7 @@ func Setup(
rsAPI roomserverAPI.FederationRoomserverAPI,
fsAPI *fedInternal.FederationInternalAPI,
keys gomatrixserverlib.JSONVerifier,
federation *gomatrixserverlib.FederationClient,
federation federationAPI.FederationClient,
userAPI userapi.FederationUserAPI,
keyAPI keyserverAPI.FederationKeyAPI,
mscCfg *config.MSCs,

2
federationapi/routing/send.go
View file

@ -85,7 +85,7 @@ func Send(
rsAPI api.FederationRoomserverAPI,
keyAPI keyapi.FederationKeyAPI,
keys gomatrixserverlib.JSONVerifier,
federation *gomatrixserverlib.FederationClient,
federation federationAPI.FederationClient,
mu *internal.MutexByRoom,
servers federationAPI.ServersInRoomProvider,
producer *producers.SyncAPIProducer,

2
federationapi/routing/send_test.go
View file

@ -8,8 +8,8 @@ import (
"time"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/test"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/gomatrixserverlib"
)

16
federationapi/routing/threepid.go
View file

@ -23,6 +23,7 @@ import (
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
federationAPI "github.com/matrix-org/dendrite/federationapi/api"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
@ -57,7 +58,7 @@ var (
func CreateInvitesFrom3PIDInvites(
req *http.Request, rsAPI api.FederationRoomserverAPI,
cfg *config.FederationAPI,
federation *gomatrixserverlib.FederationClient,
federation federationAPI.FederationClient,
userAPI userapi.FederationUserAPI,
) util.JSONResponse {
var body invites
@ -107,7 +108,7 @@ func ExchangeThirdPartyInvite(
roomID string,
rsAPI api.FederationRoomserverAPI,
cfg *config.FederationAPI,
federation *gomatrixserverlib.FederationClient,
federation federationAPI.FederationClient,
) util.JSONResponse {
var builder gomatrixserverlib.EventBuilder
if err := json.Unmarshal(request.Content(), &builder); err != nil {
@ -165,7 +166,12 @@ func ExchangeThirdPartyInvite(
// Ask the requesting server to sign the newly created event so we know it
// acknowledged it
signedEvent, err := federation.SendInvite(httpReq.Context(), request.Origin(), event)
inviteReq, err := gomatrixserverlib.NewInviteV2Request(event.Headered(verRes.RoomVersion), nil)
if err != nil {
util.GetLogger(httpReq.Context()).WithError(err).Error("failed to make invite v2 request")
return jsonerror.InternalServerError()
}
signedEvent, err := federation.SendInviteV2(httpReq.Context(), request.Origin(), inviteReq)
if err != nil {
util.GetLogger(httpReq.Context()).WithError(err).Error("federation.SendInvite failed")
return jsonerror.InternalServerError()
@ -205,7 +211,7 @@ func ExchangeThirdPartyInvite(
func createInviteFrom3PIDInvite(
ctx context.Context, rsAPI api.FederationRoomserverAPI,
cfg *config.FederationAPI,
inv invite, federation *gomatrixserverlib.FederationClient,
inv invite, federation federationAPI.FederationClient,
userAPI userapi.FederationUserAPI,
) (*gomatrixserverlib.Event, error) {
verReq := api.QueryRoomVersionForRoomRequest{RoomID: inv.RoomID}
@ -335,7 +341,7 @@ func buildMembershipEvent(
// them responded with an error.
func sendToRemoteServer(
ctx context.Context, inv invite,
federation *gomatrixserverlib.FederationClient, _ *config.FederationAPI,
federation federationAPI.FederationClient, _ *config.FederationAPI,
builder gomatrixserverlib.EventBuilder,
) (err error) {
remoteServers := make([]gomatrixserverlib.ServerName, 2)

3
federationapi/storage/interface.go
View file

@ -25,13 +25,12 @@ import (
type Database interface {
gomatrixserverlib.KeyDatabase
UpdateRoom(ctx context.Context, roomID, oldEventID, newEventID string, addHosts []types.JoinedHost, removeHosts []string) (joinedHosts []types.JoinedHost, err error)
UpdateRoom(ctx context.Context, roomID string, addHosts []types.JoinedHost, removeHosts []string, purgeRoomFirst bool) (joinedHosts []types.JoinedHost, err error)
GetJoinedHosts(ctx context.Context, roomID string) ([]types.JoinedHost, error)
GetAllJoinedHosts(ctx context.Context) ([]gomatrixserverlib.ServerName, error)
// GetJoinedHostsForRooms returns the complete set of servers in the rooms given.
GetJoinedHostsForRooms(ctx context.Context, roomIDs []string, excludeSelf bool) ([]gomatrixserverlib.ServerName, error)
PurgeRoomState(ctx context.Context, roomID string) error
StoreJSON(ctx context.Context, js string) (*shared.Receipt, error)

26
federationapi/storage/shared/storage.go
View file

@ -63,11 +63,21 @@ func (r *Receipt) String() string {
// this isn't a duplicate message.
func (d *Database) UpdateRoom(
ctx context.Context,
roomID, oldEventID, newEventID string,
roomID string,
addHosts []types.JoinedHost,
removeHosts []string,
purgeRoomFirst bool,
) (joinedHosts []types.JoinedHost, err error) {
err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
if purgeRoomFirst {
// If the event is a create event then we'll delete all of the existing
// data for the room. The only reason that a create event would be replayed
// to us in this way is if we're about to receive the entire room state.
if err = d.FederationJoinedHosts.DeleteJoinedHostsForRoom(ctx, txn, roomID); err != nil {
return fmt.Errorf("d.FederationJoinedHosts.DeleteJoinedHosts: %w", err)
}
}
joinedHosts, err = d.FederationJoinedHosts.SelectJoinedHostsWithTx(ctx, txn, roomID)
if err != nil {
return err
@ -138,20 +148,6 @@ func (d *Database) StoreJSON(
}, nil
}
func (d *Database) PurgeRoomState(
ctx context.Context, roomID string,
) error {
return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
// If the event is a create event then we'll delete all of the existing
// data for the room. The only reason that a create event would be replayed
// to us in this way is if we're about to receive the entire room state.
if err := d.FederationJoinedHosts.DeleteJoinedHostsForRoom(ctx, txn, roomID); err != nil {
return fmt.Errorf("d.FederationJoinedHosts.DeleteJoinedHosts: %w", err)
}
return nil
})
}
func (d *Database) AddServerToBlacklist(serverName gomatrixserverlib.ServerName) error {
return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
return d.FederationBlacklist.InsertBlacklist(context.TODO(), txn, serverName)

75
go.mod
View file

@ -1,22 +1,27 @@
module github.com/matrix-org/dendrite
replace github.com/nats-io/nats-server/v2 => github.com/neilalexander/nats-server/v2 v2.8.1-0.20220419100629-2278c94774f9
replace github.com/nats-io/nats-server/v2 => github.com/neilalexander/nats-server/v2 v2.8.3-0.20220513095553-73a9a246d34f
replace github.com/nats-io/nats.go => github.com/neilalexander/nats.go v1.13.1-0.20220419101051-b262d9f0be1e
require (
github.com/Arceliar/ironwood v0.0.0-20211125050254-8951369625d0
github.com/Arceliar/ironwood v0.0.0-20220306165321-319147a02d98
github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/DATA-DOG/go-sqlmock v1.5.0
github.com/MFAshby/stdemuxerhook v1.0.0
github.com/Masterminds/semver/v3 v3.1.1
github.com/Microsoft/go-winio v0.5.1 // indirect
github.com/codeclysm/extract v2.2.0+incompatible
github.com/docker/docker v20.10.14+incompatible
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/docker/docker v20.10.16+incompatible
github.com/docker/go-connections v0.4.0
github.com/ethereum/go-ethereum v1.10.17
github.com/docker/go-units v0.4.0 // indirect
github.com/frankban/quicktest v1.14.3 // indirect
github.com/getsentry/sentry-go v0.13.0
github.com/gogo/protobuf v1.3.2 // indirect
github.com/gologme/log v1.3.0
github.com/google/go-cmp v0.5.7
github.com/google/go-cmp v0.5.8
github.com/google/uuid v1.3.0
github.com/gorilla/mux v1.8.0
github.com/gorilla/websocket v1.5.0
@ -28,56 +33,62 @@ require (
github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
github.com/matrix-org/gomatrixserverlib v0.0.0-20220509120958-8d818048c34c
github.com/matrix-org/gomatrixserverlib v0.0.0-20220526140030-dcfbb70ff32d
github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
github.com/mattn/go-sqlite3 v1.14.10
github.com/mattn/go-sqlite3 v1.14.13
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/miekg/dns v1.1.49 // indirect
github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/nats-io/nats-server/v2 v2.7.4-0.20220309205833-773636c1c5bb
github.com/nats-io/nats.go v1.14.0
github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
github.com/ngrok/sqlmw v0.0.0-20211220175533-9d16fdc47b31
github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79
github.com/onsi/gomega v1.17.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
github.com/opentracing/opentracing-go v1.2.0
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/pkg/errors v0.9.1
github.com/pressly/goose v2.7.0+incompatible
github.com/prometheus/client_golang v1.12.1
github.com/prometheus/client_golang v1.12.2
github.com/sirupsen/logrus v1.8.1
github.com/stretchr/testify v1.7.0
github.com/stretchr/objx v0.2.0 // indirect
github.com/stretchr/testify v1.7.1
github.com/tidwall/gjson v1.14.1
github.com/tidwall/sjson v1.2.4
github.com/uber/jaeger-client-go v2.30.0+incompatible
github.com/uber/jaeger-lib v2.4.1+incompatible
github.com/yggdrasil-network/yggdrasil-go v0.4.3
go.uber.org/atomic v1.9.0
golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122
golang.org/x/image v0.0.0-20220321031419-a8550c1d254a
golang.org/x/mobile v0.0.0-20220407111146-e579adbbc4a2
golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e
golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9
golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd
golang.org/x/net v0.0.0-20220524220425-1d687d428aca
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
golang.org/x/term v0.0.0-20220526004731-065cf7ba2467
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/h2non/bimg.v1 v1.1.9
gopkg.in/yaml.v2 v2.4.0
gopkg.in/yaml.v3 v3.0.0 // indirect
gotest.tools/v3 v3.0.3 // indirect
nhooyr.io/websocket v1.8.7
)
require github.com/ethereum/go-ethereum v1.10.18
require (
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
github.com/Microsoft/go-winio v0.5.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/btcsuite/btcd/btcec/v2 v2.1.2 // indirect
github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cheekybits/genny v1.0.0 // indirect
github.com/containerd/containerd v1.6.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/frankban/quicktest v1.14.3 // indirect
github.com/fsnotify/fsnotify v1.4.9 // indirect
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/juju/errors v0.0.0-20220203013757-bd733f3c86b9 // indirect
github.com/klauspost/compress v1.14.4 // indirect
@ -85,37 +96,25 @@ require (
github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
github.com/marten-seemann/qtls-go1-17 v0.1.1 // indirect
github.com/marten-seemann/qtls-go1-18 v0.1.1 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/miekg/dns v1.1.31 // indirect
github.com/minio/highwayhash v1.0.2 // indirect
github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/nats-io/jwt/v2 v2.2.1-0.20220330180145-442af02fd36a // indirect
github.com/nats-io/nkeys v0.3.0 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/nxadm/tail v1.4.8 // indirect
github.com/onsi/ginkgo v1.16.4 // indirect
github.com/onsi/gomega v1.15.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.2 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/stretchr/objx v0.2.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
golang.org/x/mod v0.4.2 // indirect
golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect
golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b // indirect
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098 // indirect
golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect
google.golang.org/grpc v1.43.0 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/macaroon.v2 v2.1.0 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
)
go 1.17

876
go.sum
View file

File diff suppressed because it is too large Load diff

2
internal/caching/cache_typing_test.go
View file

@ -20,7 +20,7 @@ import (
"testing"
"time"
"github.com/matrix-org/dendrite/internal/test"
"github.com/matrix-org/dendrite/test"
)
func TestEDUCache(t *testing.T) {

4
internal/sqlutil/trace.go
View file

@ -37,7 +37,7 @@ type traceInterceptor struct {
sqlmw.NullInterceptor
}
func (in *traceInterceptor) StmtQueryContext(ctx context.Context, stmt driver.StmtQueryContext, query string, args []driver.NamedValue) (driver.Rows, error) {
func (in *traceInterceptor) StmtQueryContext(ctx context.Context, stmt driver.StmtQueryContext, query string, args []driver.NamedValue) (context.Context, driver.Rows, error) {
startedAt := time.Now()
rows, err := stmt.QueryContext(ctx, args)
@ -45,7 +45,7 @@ func (in *traceInterceptor) StmtQueryContext(ctx context.Context, stmt driver.St
logrus.WithField("duration", time.Since(startedAt)).WithField(logrus.ErrorKey, err).Debug("executed sql query ", query, " args: ", args)
return rows, err
return ctx, rows, err
}
func (in *traceInterceptor) StmtExecContext(ctx context.Context, stmt driver.StmtExecContext, query string, args []driver.NamedValue) (driver.Result, error) {

158
internal/test/client.go
View file

@ -1,158 +0,0 @@
// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (
"crypto/tls"
"fmt"
"io"
"io/ioutil"
"net/http"
"sync"
"time"
"github.com/matrix-org/gomatrixserverlib"
)
// Request contains the information necessary to issue a request and test its result
type Request struct {
Req *http.Request
WantedBody string
WantedStatusCode int
LastErr *LastRequestErr
}
// LastRequestErr is a synchronised error wrapper
// Useful for obtaining the last error from a set of requests
type LastRequestErr struct {
sync.Mutex
Err error
}
// Set sets the error
func (r *LastRequestErr) Set(err error) {
r.Lock()
defer r.Unlock()
r.Err = err
}
// Get gets the error
func (r *LastRequestErr) Get() error {
r.Lock()
defer r.Unlock()
return r.Err
}
// CanonicalJSONInput canonicalises a slice of JSON strings
// Useful for test input
func CanonicalJSONInput(jsonData []string) []string {
for i := range jsonData {
jsonBytes, err := gomatrixserverlib.CanonicalJSON([]byte(jsonData[i]))
if err != nil && err != io.EOF {
panic(err)
}
jsonData[i] = string(jsonBytes)
}
return jsonData
}
// Do issues a request and checks the status code and body of the response
func (r *Request) Do() (err error) {
client := &http.Client{
Timeout: 5 * time.Second,
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
},
}
res, err := client.Do(r.Req)
if err != nil {
return err
}
defer (func() { err = res.Body.Close() })()
if res.StatusCode != r.WantedStatusCode {
return fmt.Errorf("incorrect status code. Expected: %d Got: %d", r.WantedStatusCode, res.StatusCode)
}
if r.WantedBody != "" {
resBytes, err := ioutil.ReadAll(res.Body)
if err != nil {
return err
}
jsonBytes, err := gomatrixserverlib.CanonicalJSON(resBytes)
if err != nil {
return err
}
if string(jsonBytes) != r.WantedBody {
return fmt.Errorf("returned wrong bytes. Expected:\n%s\n\nGot:\n%s", r.WantedBody, string(jsonBytes))
}
}
return nil
}
// DoUntilSuccess blocks and repeats the same request until the response returns the desired status code and body.
// It then closes the given channel and returns.
func (r *Request) DoUntilSuccess(done chan error) {
r.LastErr = &LastRequestErr{}
for {
if err := r.Do(); err != nil {
r.LastErr.Set(err)
time.Sleep(1 * time.Second) // don't tightloop
continue
}
close(done)
return
}
}
// Run repeatedly issues a request until success, error or a timeout is reached
func (r *Request) Run(label string, timeout time.Duration, serverCmdChan chan error) {
fmt.Printf("==TESTING== %v (timeout: %v)\n", label, timeout)
done := make(chan error, 1)
// We need to wait for the server to:
// - have connected to the database
// - have created the tables
// - be listening on the given port
go r.DoUntilSuccess(done)
// wait for one of:
// - the test to pass (done channel is closed)
// - the server to exit with an error (error sent on serverCmdChan)
// - our test timeout to expire
// We don't need to clean up since the main() function handles that in the event we panic
select {
case <-time.After(timeout):
fmt.Printf("==TESTING== %v TIMEOUT\n", label)
if reqErr := r.LastErr.Get(); reqErr != nil {
fmt.Println("Last /sync request error:")
fmt.Println(reqErr)
}
panic(fmt.Sprintf("%v server timed out", label))
case err := <-serverCmdChan:
if err != nil {
fmt.Println("=============================================================================================")
fmt.Printf("%v server failed to run. If failing with 'pq: password authentication failed for user' try:", label)
fmt.Println(" export PGHOST=/var/run/postgresql")
fmt.Println("=============================================================================================")
panic(err)
}
case <-done:
fmt.Printf("==TESTING== %v PASSED\n", label)
}
}

76
internal/test/kafka.go
View file

@ -1,76 +0,0 @@
// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (
"io"
"os/exec"
"path/filepath"
"strings"
)
// KafkaExecutor executes kafka scripts.
type KafkaExecutor struct {
// The location of Zookeeper. Typically this is `localhost:2181`.
ZookeeperURI string
// The directory where Kafka is installed to. Used to locate kafka scripts.
KafkaDirectory string
// The location of the Kafka logs. Typically this is `localhost:9092`.
KafkaURI string
// Where stdout and stderr should be written to. Typically this is `os.Stderr`.
OutputWriter io.Writer
}
// CreateTopic creates a new kafka topic. This is created with a single partition.
func (e *KafkaExecutor) CreateTopic(topic string) error {
cmd := exec.Command(
filepath.Join(e.KafkaDirectory, "bin", "kafka-topics.sh"),
"--create",
"--zookeeper", e.ZookeeperURI,
"--replication-factor", "1",
"--partitions", "1",
"--topic", topic,
)
cmd.Stdout = e.OutputWriter
cmd.Stderr = e.OutputWriter
return cmd.Run()
}
// WriteToTopic writes data to a kafka topic.
func (e *KafkaExecutor) WriteToTopic(topic string, data []string) error {
cmd := exec.Command(
filepath.Join(e.KafkaDirectory, "bin", "kafka-console-producer.sh"),
"--broker-list", e.KafkaURI,
"--topic", topic,
)
cmd.Stdout = e.OutputWriter
cmd.Stderr = e.OutputWriter
cmd.Stdin = strings.NewReader(strings.Join(data, "\n"))
return cmd.Run()
}
// DeleteTopic deletes a given kafka topic if it exists.
func (e *KafkaExecutor) DeleteTopic(topic string) error {
cmd := exec.Command(
filepath.Join(e.KafkaDirectory, "bin", "kafka-topics.sh"),
"--delete",
"--if-exists",
"--zookeeper", e.ZookeeperURI,
"--topic", topic,
)
cmd.Stderr = e.OutputWriter
cmd.Stdout = e.OutputWriter
return cmd.Run()
}

152
internal/test/server.go
View file

@ -1,152 +0,0 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (
"context"
"fmt"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"testing"
"github.com/matrix-org/dendrite/setup/config"
)
// Defaulting allows assignment of string variables with a fallback default value
// Useful for use with os.Getenv() for example
func Defaulting(value, defaultValue string) string {
if value == "" {
value = defaultValue
}
return value
}
// CreateDatabase creates a new database, dropping it first if it exists
func CreateDatabase(command string, args []string, database string) error {
cmd := exec.Command(command, args...)
cmd.Stdin = strings.NewReader(
fmt.Sprintf("DROP DATABASE IF EXISTS %s; CREATE DATABASE %s;", database, database),
)
// Send stdout and stderr to our stderr so that we see error messages from
// the psql process
cmd.Stdout = os.Stderr
cmd.Stderr = os.Stderr
return cmd.Run()
}
// CreateBackgroundCommand creates an executable command
// The Cmd being executed is returned. A channel is also returned,
// which will have any termination errors sent down it, followed immediately by the channel being closed.
func CreateBackgroundCommand(command string, args []string) (*exec.Cmd, chan error) {
cmd := exec.Command(command, args...)
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stderr
if err := cmd.Start(); err != nil {
panic("failed to start server: " + err.Error())
}
cmdChan := make(chan error, 1)
go func() {
cmdChan <- cmd.Wait()
close(cmdChan)
}()
return cmd, cmdChan
}
// InitDatabase creates the database and config file needed for the server to run
func InitDatabase(postgresDatabase, postgresContainerName string, databases []string) {
if len(databases) > 0 {
var dbCmd string
var dbArgs []string
if postgresContainerName == "" {
dbCmd = "psql"
dbArgs = []string{postgresDatabase}
} else {
dbCmd = "docker"
dbArgs = []string{
"exec", "-i", postgresContainerName, "psql", "-U", "postgres", postgresDatabase,
}
}
for _, database := range databases {
if err := CreateDatabase(dbCmd, dbArgs, database); err != nil {
panic(err)
}
}
}
}
// StartProxy creates a reverse proxy
func StartProxy(bindAddr string, cfg *config.Dendrite) (*exec.Cmd, chan error) {
proxyArgs := []string{
"--bind-address", bindAddr,
"--sync-api-server-url", "http://" + string(cfg.SyncAPI.InternalAPI.Connect),
"--client-api-server-url", "http://" + string(cfg.ClientAPI.InternalAPI.Connect),
"--media-api-server-url", "http://" + string(cfg.MediaAPI.InternalAPI.Connect),
"--tls-cert", "server.crt",
"--tls-key", "server.key",
}
return CreateBackgroundCommand(
filepath.Join(filepath.Dir(os.Args[0]), "client-api-proxy"),
proxyArgs,
)
}
// ListenAndServe will listen on a random high-numbered port and attach the given router.
// Returns the base URL to send requests to. Call `cancel` to shutdown the server, which will block until it has closed.
func ListenAndServe(t *testing.T, router http.Handler, useTLS bool) (apiURL string, cancel func()) {
listener, err := net.Listen("tcp", ":0")
if err != nil {
t.Fatalf("failed to listen: %s", err)
}
port := listener.Addr().(*net.TCPAddr).Port
srv := http.Server{}
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
srv.Handler = router
var err error
if useTLS {
certFile := filepath.Join(os.TempDir(), "dendrite.cert")
keyFile := filepath.Join(os.TempDir(), "dendrite.key")
err = NewTLSKey(keyFile, certFile)
if err != nil {
t.Logf("failed to generate tls key/cert: %s", err)
return
}
err = srv.ServeTLS(listener, certFile, keyFile)
} else {
err = srv.Serve(listener)
}
if err != nil && err != http.ErrServerClosed {
t.Logf("Listen failed: %s", err)
}
}()
secure := ""
if useTLS {
secure = "s"
}
return fmt.Sprintf("http%s://localhost:%d", secure, port), func() {
_ = srv.Shutdown(context.Background())
wg.Wait()
}
}

2
internal/version.go
View file

@ -17,7 +17,7 @@ var build string
const (
VersionMajor = 0
VersionMinor = 8
VersionPatch = 4
VersionPatch = 6
VersionTag = "" // example: "rc1"
)

4
keyserver/internal/device_list_update.go
View file

@ -84,7 +84,7 @@ type DeviceListUpdater struct {
db DeviceListUpdaterDatabase
api DeviceListUpdaterAPI
producer KeyChangeProducer
fedClient fedsenderapi.FederationClient
fedClient fedsenderapi.KeyserverFederationAPI
workerChans []chan gomatrixserverlib.ServerName
// When device lists are stale for a user, they get inserted into this map with a channel which `Update` will
@ -127,7 +127,7 @@ type KeyChangeProducer interface {
// NewDeviceListUpdater creates a new updater which fetches fresh device lists when they go stale.
func NewDeviceListUpdater(
db DeviceListUpdaterDatabase, api DeviceListUpdaterAPI, producer KeyChangeProducer,
fedClient fedsenderapi.FederationClient, numWorkers int,
fedClient fedsenderapi.KeyserverFederationAPI, numWorkers int,
) *DeviceListUpdater {
return &DeviceListUpdater{
userIDToMutex: make(map[string]*sync.Mutex),

2
keyserver/internal/internal.go
View file

@ -37,7 +37,7 @@ import (
type KeyInternalAPI struct {
DB storage.Database
ThisServer gomatrixserverlib.ServerName
FedClient fedsenderapi.FederationClient
FedClient fedsenderapi.KeyserverFederationAPI
UserAPI userapi.KeyserverUserAPI
Producer *producers.KeyChange
Updater *DeviceListUpdater

2
keyserver/keyserver.go
View file

@ -37,7 +37,7 @@ func AddInternalRoutes(router *mux.Router, intAPI api.KeyInternalAPI) {
// NewInternalAPI returns a concerete implementation of the internal API. Callers
// can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
func NewInternalAPI(
base *base.BaseDendrite, cfg *config.KeyServer, fedClient fedsenderapi.FederationClient,
base *base.BaseDendrite, cfg *config.KeyServer, fedClient fedsenderapi.KeyserverFederationAPI,
) api.KeyInternalAPI {
js, _ := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)

2
roomserver/api/api.go
View file

@ -183,6 +183,8 @@ type FederationRoomserverAPI interface {
QueryMissingEvents(ctx context.Context, req *QueryMissingEventsRequest, res *QueryMissingEventsResponse) error
// Query whether a server is allowed to see an event
QueryServerAllowedToSeeEvent(ctx context.Context, req *QueryServerAllowedToSeeEventRequest, res *QueryServerAllowedToSeeEventResponse) error
QueryRoomsForUser(ctx context.Context, req *QueryRoomsForUserRequest, res *QueryRoomsForUserResponse) error
QueryRestrictedJoinAllowed(ctx context.Context, req *QueryRestrictedJoinAllowedRequest, res *QueryRestrictedJoinAllowedResponse) error
PerformInboundPeek(ctx context.Context, req *PerformInboundPeekRequest, res *PerformInboundPeekResponse) error
PerformInvite(ctx context.Context, req *PerformInviteRequest, res *PerformInviteResponse) error
// Query a given amount (or less) of events prior to a given set of events.

10
roomserver/api/api_trace.go
View file

@ -354,6 +354,16 @@ func (t *RoomserverInternalAPITrace) QueryAuthChain(
return err
}
func (t *RoomserverInternalAPITrace) QueryRestrictedJoinAllowed(
ctx context.Context,
request *QueryRestrictedJoinAllowedRequest,
response *QueryRestrictedJoinAllowedResponse,
) error {
err := t.Impl.QueryRestrictedJoinAllowed(ctx, request, response)
util.GetLogger(ctx).WithError(err).Infof("QueryRestrictedJoinAllowed req=%+v res=%+v", js(request), js(response))
return err
}
func js(thing interface{}) string {
b, err := json.Marshal(thing)
if err != nil {

20
roomserver/api/query.go
View file

@ -348,6 +348,26 @@ type QueryServerBannedFromRoomResponse struct {
Banned bool `json:"banned"`
}
type QueryRestrictedJoinAllowedRequest struct {
UserID string `json:"user_id"`
RoomID string `json:"room_id"`
}
type QueryRestrictedJoinAllowedResponse struct {
// True if the room membership is restricted by the join rule being set to "restricted"
Restricted bool `json:"restricted"`
// True if our local server is joined to all of the allowed rooms specified in the "allow"
// key of the join rule, false if we are missing from some of them and therefore can't
// reliably decide whether or not we can satisfy the join
Resident bool `json:"resident"`
// True if the restricted join is allowed because we found the membership in one of the
// allowed rooms from the join rule, false if not
Allowed bool `json:"allowed"`
// Contains the user ID of the selected user ID that has power to issue invites, this will
// get populated into the "join_authorised_via_users_server" content in the membership
AuthorisedVia string `json:"authorised_via,omitempty"`
}
// MarshalJSON stringifies the room ID and StateKeyTuple keys so they can be sent over the wire in HTTP API mode.
func (r *QueryBulkStateContentResponse) MarshalJSON() ([]byte, error) {
se := make(map[string]string)

4
roomserver/internal/input/input_events.go
View file

@ -535,8 +535,6 @@ func (r *Inputer) calculateAndSetState(
roomState := state.NewStateResolution(updater, roomInfo)
if input.HasState {
stateAtEvent.Overwrite = true
// We've been told what the state at the event is so we don't need to calculate it.
// Check that those state events are in the database and store the state.
var entries []types.StateEntry
@ -549,8 +547,6 @@ func (r *Inputer) calculateAndSetState(
return fmt.Errorf("updater.AddState: %w", err)
}
} else {
stateAtEvent.Overwrite = false
// We haven't been told what the state at the event is so we need to calculate it from the prev_events
if stateAtEvent.BeforeStateSnapshotNID, err = roomState.CalculateAndStoreStateBeforeEvent(ctx, event, isRejected); err != nil {
return fmt.Errorf("roomState.CalculateAndStoreStateBeforeEvent: %w", err)

44
roomserver/internal/input/input_latest_events.go
View file

@ -27,6 +27,7 @@ import (
"github.com/matrix-org/dendrite/roomserver/types"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/sirupsen/logrus"
)
// updateLatestEvents updates the list of latest events for this room in the database and writes the
@ -101,8 +102,8 @@ type latestEventsUpdater struct {
// The eventID of the event that was processed before this one.
lastEventIDSent string
// The latest events in the room after processing this event.
oldLatest []types.StateAtEventAndReference
latest []types.StateAtEventAndReference
oldLatest types.StateAtEventAndReferences
latest types.StateAtEventAndReferences
// The state entries removed from and added to the current state of the
// room as a result of processing this event. They are sorted lists.
removed []types.StateEntry
@ -125,7 +126,7 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error {
// state snapshot from somewhere else, e.g. a federated room join,
// then start with an empty set - none of the forward extremities
// that we knew about before matter anymore.
u.oldLatest = []types.StateAtEventAndReference{}
u.oldLatest = types.StateAtEventAndReferences{}
if !u.rewritesState {
u.oldStateNID = u.updater.CurrentStateSnapshotNID()
u.oldLatest = u.updater.LatestEvents()
@ -233,12 +234,19 @@ func (u *latestEventsUpdater) latestState() error {
}
}
// Get a list of the current latest events. This may or may not
// include the new event from the input path, depending on whether
// it is a forward extremity or not.
latestStateAtEvents := make([]types.StateAtEvent, len(u.latest))
for i := range u.latest {
latestStateAtEvents[i] = u.latest[i].StateAtEvent
// Take the old set of extremities and the new set of extremities and
// mash them together into a list. This may or may not include the new event
// from the input path, depending on whether it became a forward extremity
// or not. We'll then run state resolution across all of them to determine
// the new current state of the room. Including the old extremities here
// ensures that new forward extremities with bad state snapshots (from
// possible malicious actors) can't completely corrupt the room state
// away from what it was before.
combinedExtremities := types.StateAtEventAndReferences(append(u.oldLatest, u.latest...))
combinedExtremities = combinedExtremities[:util.SortAndUnique(combinedExtremities)]
latestStateAtEvents := make([]types.StateAtEvent, len(combinedExtremities))
for i := range combinedExtremities {
latestStateAtEvents[i] = combinedExtremities[i].StateAtEvent
}
// Takes the NIDs of the latest events and creates a state snapshot
@ -251,13 +259,6 @@ func (u *latestEventsUpdater) latestState() error {
return fmt.Errorf("roomState.CalculateAndStoreStateAfterEvents: %w", err)
}
// If we are overwriting the state then we should make sure that we
// don't send anything out over federation again, it will very likely
// be a repeat.
if u.stateAtEvent.Overwrite {
u.sendAsServer = ""
}
// Now that we have a new state snapshot based on the latest events,
// we can compare that new snapshot to the previous one and see what
// has changed. This gives us one list of removed state events and
@ -270,6 +271,17 @@ func (u *latestEventsUpdater) latestState() error {
return fmt.Errorf("roomState.DifferenceBetweenStateSnapshots: %w", err)
}
if removed := len(u.removed) - len(u.added); removed > 0 {
logrus.WithFields(logrus.Fields{
"event_id": u.event.EventID(),
"room_id": u.event.RoomID(),
"old_state_nid": u.oldStateNID,
"new_state_nid": u.newStateNID,
"old_latest": u.oldLatest.EventIDs(),
"new_latest": u.latest.EventIDs(),
}).Errorf("Unexpected state deletion (removing %d events)", removed)
}
// Also work out the state before the event removes and the event
// adds.
u.stateBeforeEventRemoves, u.stateBeforeEventAdds, err = roomState.DifferenceBetweeenStateSnapshots(

4
roomserver/internal/input/input_test.go
View file

@ -12,7 +12,7 @@ import (
"github.com/matrix-org/dendrite/roomserver/storage"
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/test/testrig"
"github.com/matrix-org/gomatrixserverlib"
"github.com/nats-io/nats.go"
)
@ -22,7 +22,7 @@ var jc *nats.Conn
func TestMain(m *testing.M) {
var b *base.BaseDendrite
b, js, jc = test.Base(nil)
b, js, jc = testrig.Base(nil)
code := m.Run()
b.ShutdownDendrite()
b.WaitForComponentsToFinish()

34
roomserver/internal/perform/perform_join.go
View file

@ -24,6 +24,7 @@ import (
"github.com/getsentry/sentry-go"
fsAPI "github.com/matrix-org/dendrite/federationapi/api"
"github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/roomserver/api"
rsAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/roomserver/internal/helpers"
"github.com/matrix-org/dendrite/roomserver/internal/input"
@ -160,6 +161,7 @@ func (r *Joiner) performJoinRoomByAlias(
}
// TODO: Break this function up a bit
// nolint:gocyclo
func (r *Joiner) performJoinRoomByID(
ctx context.Context,
req *rsAPI.PerformJoinRequest,
@ -210,6 +212,11 @@ func (r *Joiner) performJoinRoomByID(
req.Content = map[string]interface{}{}
}
req.Content["membership"] = gomatrixserverlib.Join
if authorisedVia, aerr := r.populateAuthorisedViaUserForRestrictedJoin(ctx, req); aerr != nil {
return "", "", aerr
} else if authorisedVia != "" {
req.Content["join_authorised_via_users_server"] = authorisedVia
}
if err = eb.SetContent(req.Content); err != nil {
return "", "", fmt.Errorf("eb.SetContent: %w", err)
}
@ -350,6 +357,33 @@ func (r *Joiner) performFederatedJoinRoomByID(
return fedRes.JoinedVia, nil
}
func (r *Joiner) populateAuthorisedViaUserForRestrictedJoin(
ctx context.Context,
joinReq *rsAPI.PerformJoinRequest,
) (string, error) {
req := &api.QueryRestrictedJoinAllowedRequest{
UserID: joinReq.UserID,
RoomID: joinReq.RoomIDOrAlias,
}
res := &api.QueryRestrictedJoinAllowedResponse{}
if err := r.Queryer.QueryRestrictedJoinAllowed(ctx, req, res); err != nil {
return "", fmt.Errorf("r.Queryer.QueryRestrictedJoinAllowed: %w", err)
}
if !res.Restricted {
return "", nil
}
if !res.Resident {
return "", nil
}
if !res.Allowed {
return "", &rsAPI.PerformError{
Code: rsAPI.PerformErrorNotAllowed,
Msg: fmt.Sprintf("The join to room %s was not allowed.", joinReq.RoomIDOrAlias),
}
}
return res.AuthorisedVia, nil
}
func buildEvent(
ctx context.Context, db storage.Database, cfg *config.Global, builder *gomatrixserverlib.EventBuilder,
) (*gomatrixserverlib.HeaderedEvent, *rsAPI.QueryLatestEventsAndStateResponse, error) {

126
roomserver/internal/query/query.go
View file

@ -16,6 +16,7 @@ package query
import (
"context"
"encoding/json"
"errors"
"fmt"
@ -757,3 +758,128 @@ func (r *Queryer) QueryAuthChain(ctx context.Context, req *api.QueryAuthChainReq
res.AuthChain = hchain
return nil
}
// nolint:gocyclo
func (r *Queryer) QueryRestrictedJoinAllowed(ctx context.Context, req *api.QueryRestrictedJoinAllowedRequest, res *api.QueryRestrictedJoinAllowedResponse) error {
// Look up if we know anything about the room. If it doesn't exist
// or is a stub entry then we can't do anything.
roomInfo, err := r.DB.RoomInfo(ctx, req.RoomID)
if err != nil {
return fmt.Errorf("r.DB.RoomInfo: %w", err)
}
if roomInfo == nil || roomInfo.IsStub {
return nil // fmt.Errorf("room %q doesn't exist or is stub room", req.RoomID)
}
// If the room version doesn't allow restricted joins then don't
// try to process any further.
allowRestrictedJoins, err := roomInfo.RoomVersion.MayAllowRestrictedJoinsInEventAuth()
if err != nil {
return fmt.Errorf("roomInfo.RoomVersion.AllowRestrictedJoinsInEventAuth: %w", err)
} else if !allowRestrictedJoins {
return nil
}
// Get the join rules to work out if the join rule is "restricted".
joinRulesEvent, err := r.DB.GetStateEvent(ctx, req.RoomID, gomatrixserverlib.MRoomJoinRules, "")
if err != nil {
return fmt.Errorf("r.DB.GetStateEvent: %w", err)
}
var joinRules gomatrixserverlib.JoinRuleContent
if err = json.Unmarshal(joinRulesEvent.Content(), &joinRules); err != nil {
return fmt.Errorf("json.Unmarshal: %w", err)
}
// If the join rule isn't "restricted" then there's nothing more to do.
res.Restricted = joinRules.JoinRule == gomatrixserverlib.Restricted
if !res.Restricted {
return nil
}
// Start off by populating the "resident" flag in the response. If we
// come across any rooms in the request that are missing, we will unset
// the flag.
res.Resident = true
// If the user is already invited to the room then the join is allowed
// but we don't specify an authorised via user, since the event auth
// will allow the join anyway.
var pending bool
if pending, _, _, err = helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID); err != nil {
return fmt.Errorf("helpers.IsInvitePending: %w", err)
} else if pending {
res.Allowed = true
return nil
}
// We need to get the power levels content so that we can determine which
// users in the room are entitled to issue invites. We need to use one of
// these users as the authorising user.
powerLevelsEvent, err := r.DB.GetStateEvent(ctx, req.RoomID, gomatrixserverlib.MRoomPowerLevels, "")
if err != nil {
return fmt.Errorf("r.DB.GetStateEvent: %w", err)
}
var powerLevels gomatrixserverlib.PowerLevelContent
if err = json.Unmarshal(powerLevelsEvent.Content(), &powerLevels); err != nil {
return fmt.Errorf("json.Unmarshal: %w", err)
}
// Step through the join rules and see if the user matches any of them.
for _, rule := range joinRules.Allow {
// We only understand "m.room_membership" rules at this point in
// time, so skip any rule that doesn't match those.
if rule.Type != gomatrixserverlib.MRoomMembership {
continue
}
// See if the room exists. If it doesn't exist or if it's a stub
// room entry then we can't check memberships.
targetRoomInfo, err := r.DB.RoomInfo(ctx, rule.RoomID)
if err != nil || targetRoomInfo == nil || targetRoomInfo.IsStub {
res.Resident = false
continue
}
// First of all work out if *we* are still in the room, otherwise
// it's possible that the memberships will be out of date.
isIn, err := r.DB.GetLocalServerInRoom(ctx, targetRoomInfo.RoomNID)
if err != nil || !isIn {
// If we aren't in the room, we can no longer tell if the room
// memberships are up-to-date.
res.Resident = false
continue
}
// At this point we're happy that we are in the room, so now let's
// see if the target user is in the room.
_, isIn, _, err = r.DB.GetMembership(ctx, targetRoomInfo.RoomNID, req.UserID)
if err != nil {
continue
}
// If the user is not in the room then we will skip them.
if !isIn {
continue
}
// The user is in the room, so now we will need to authorise the
// join using the user ID of one of our own users in the room. Pick
// one.
joinNIDs, err := r.DB.GetMembershipEventNIDsForRoom(ctx, targetRoomInfo.RoomNID, true, true)
if err != nil || len(joinNIDs) == 0 {
// There should always be more than one join NID at this point
// because we are gated behind GetLocalServerInRoom, but y'know,
// sometimes strange things happen.
continue
}
// For each of the joined users, let's see if we can get a valid
// membership event.
for _, joinNID := range joinNIDs {
events, err := r.DB.Events(ctx, []types.EventNID{joinNID})
if err != nil || len(events) != 1 {
continue
}
event := events[0]
if event.Type() != gomatrixserverlib.MRoomMember || event.StateKey() == nil {
continue // shouldn't happen
}
// Only users that have the power to invite should be chosen.
if powerLevels.UserLevel(*event.StateKey()) < powerLevels.Invite {
continue
}
res.Resident = true
res.Allowed = true
res.AuthorisedVia = *event.StateKey()
return nil
}
}
return nil
}

2
roomserver/internal/query/query_test.go
View file

@ -19,8 +19,8 @@ import (
"encoding/json"
"testing"
"github.com/matrix-org/dendrite/internal/test"
"github.com/matrix-org/dendrite/roomserver/types"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/gomatrixserverlib"
)

11
roomserver/inthttp/client.go
View file

@ -61,6 +61,7 @@ const (
RoomserverQueryKnownUsersPath = "/roomserver/queryKnownUsers"
RoomserverQueryServerBannedFromRoomPath = "/roomserver/queryServerBannedFromRoom"
RoomserverQueryAuthChainPath = "/roomserver/queryAuthChain"
RoomserverQueryRestrictedJoinAllowed = "/roomserver/queryRestrictedJoinAllowed"
)
type httpRoomserverInternalAPI struct {
@ -557,6 +558,16 @@ func (h *httpRoomserverInternalAPI) QueryServerBannedFromRoom(
return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
}
func (h *httpRoomserverInternalAPI) QueryRestrictedJoinAllowed(
ctx context.Context, req *api.QueryRestrictedJoinAllowedRequest, res *api.QueryRestrictedJoinAllowedResponse,
) error {
span, ctx := opentracing.StartSpanFromContext(ctx, "QueryRestrictedJoinAllowed")
defer span.Finish()
apiURL := h.roomserverURL + RoomserverQueryRestrictedJoinAllowed
return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
}
func (h *httpRoomserverInternalAPI) PerformForget(ctx context.Context, req *api.PerformForgetRequest, res *api.PerformForgetResponse) error {
span, ctx := opentracing.StartSpanFromContext(ctx, "PerformForget")
defer span.Finish()

13
roomserver/inthttp/server.go
View file

@ -472,4 +472,17 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) {
return util.JSONResponse{Code: http.StatusOK, JSON: &response}
}),
)
internalAPIMux.Handle(RoomserverQueryRestrictedJoinAllowed,
httputil.MakeInternalAPI("queryRestrictedJoinAllowed", func(req *http.Request) util.JSONResponse {
request := api.QueryRestrictedJoinAllowedRequest{}
response := api.QueryRestrictedJoinAllowedResponse{}
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
return util.MessageResponse(http.StatusBadRequest, err.Error())
}
if err := r.QueryRestrictedJoinAllowed(req.Context(), &request, &response); err != nil {
return util.ErrorResponse(err)
}
return util.JSONResponse{Code: http.StatusOK, JSON: &response}
}),
)
}

6
roomserver/storage/postgres/events_table.go
View file

@ -264,11 +264,11 @@ func (s *eventStatements) BulkSelectStateEventByNID(
ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID,
stateKeyTuples []types.StateKeyTuple,
) ([]types.StateEntry, error) {
tuples := stateKeyTupleSorter(stateKeyTuples)
tuples := types.StateKeyTupleSorter(stateKeyTuples)
sort.Sort(tuples)
eventTypeNIDArray, eventStateKeyNIDArray := tuples.typesAndStateKeysAsArrays()
eventTypeNIDArray, eventStateKeyNIDArray := tuples.TypesAndStateKeysAsArrays()
stmt := sqlutil.TxStmt(txn, s.bulkSelectStateEventByNIDStmt)
rows, err := stmt.QueryContext(ctx, eventNIDsAsArray(eventNIDs), eventTypeNIDArray, eventStateKeyNIDArray)
rows, err := stmt.QueryContext(ctx, eventNIDsAsArray(eventNIDs), pq.Int64Array(eventTypeNIDArray), pq.Int64Array(eventStateKeyNIDArray))
if err != nil {
return nil, err
}

6
roomserver/storage/postgres/room_aliases_table.go
View file

@ -61,12 +61,12 @@ type roomAliasesStatements struct {
deleteRoomAliasStmt *sql.Stmt
}
func createRoomAliasesTable(db *sql.DB) error {
func CreateRoomAliasesTable(db *sql.DB) error {
_, err := db.Exec(roomAliasesSchema)
return err
}
func prepareRoomAliasesTable(db *sql.DB) (tables.RoomAliases, error) {
func PrepareRoomAliasesTable(db *sql.DB) (tables.RoomAliases, error) {
s := &roomAliasesStatements{}
return s, sqlutil.StatementList{
@ -108,8 +108,8 @@ func (s *roomAliasesStatements) SelectAliasesFromRoomID(
defer internal.CloseAndLogIfError(ctx, rows, "selectAliasesFromRoomID: rows.close() failed")
var aliases []string
var alias string
for rows.Next() {
var alias string
if err = rows.Scan(&alias); err != nil {
return nil, err
}

16
roomserver/storage/postgres/rooms_table.go
View file

@ -95,12 +95,12 @@ type roomStatements struct {
bulkSelectRoomNIDsStmt *sql.Stmt
}
func createRoomsTable(db *sql.DB) error {
func CreateRoomsTable(db *sql.DB) error {
_, err := db.Exec(roomsSchema)
return err
}
func prepareRoomsTable(db *sql.DB) (tables.Rooms, error) {
func PrepareRoomsTable(db *sql.DB) (tables.Rooms, error) {
s := &roomStatements{}
return s, sqlutil.StatementList{
@ -117,7 +117,7 @@ func prepareRoomsTable(db *sql.DB) (tables.Rooms, error) {
}.Prepare(db)
}
func (s *roomStatements) SelectRoomIDs(ctx context.Context, txn *sql.Tx) ([]string, error) {
func (s *roomStatements) SelectRoomIDsWithEvents(ctx context.Context, txn *sql.Tx) ([]string, error) {
stmt := sqlutil.TxStmt(txn, s.selectRoomIDsStmt)
rows, err := stmt.QueryContext(ctx)
if err != nil {
@ -125,8 +125,8 @@ func (s *roomStatements) SelectRoomIDs(ctx context.Context, txn *sql.Tx) ([]stri
}
defer internal.CloseAndLogIfError(ctx, rows, "selectRoomIDsStmt: rows.close() failed")
var roomIDs []string
var roomID string
for rows.Next() {
var roomID string
if err = rows.Scan(&roomID); err != nil {
return nil, err
}
@ -231,9 +231,9 @@ func (s *roomStatements) SelectRoomVersionsForRoomNIDs(
}
defer internal.CloseAndLogIfError(ctx, rows, "selectRoomVersionsForRoomNIDsStmt: rows.close() failed")
result := make(map[types.RoomNID]gomatrixserverlib.RoomVersion)
var roomNID types.RoomNID
var roomVersion gomatrixserverlib.RoomVersion
for rows.Next() {
var roomNID types.RoomNID
var roomVersion gomatrixserverlib.RoomVersion
if err = rows.Scan(&roomNID, &roomVersion); err != nil {
return nil, err
}
@ -254,8 +254,8 @@ func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roo
}
defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectRoomIDsStmt: rows.close() failed")
var roomIDs []string
var roomID string
for rows.Next() {
var roomID string
if err = rows.Scan(&roomID); err != nil {
return nil, err
}
@ -276,8 +276,8 @@ func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, ro
}
defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectRoomNIDsStmt: rows.close() failed")
var roomNIDs []types.RoomNID
var roomNID types.RoomNID
for rows.Next() {
var roomNID types.RoomNID
if err = rows.Scan(&roomNID); err != nil {
return nil, err
}

53
roomserver/storage/postgres/state_block_table.go
View file

@ -19,7 +19,6 @@ import (
"context"
"database/sql"
"fmt"
"sort"
"github.com/lib/pq"
"github.com/matrix-org/dendrite/internal"
@ -71,12 +70,12 @@ type stateBlockStatements struct {
bulkSelectStateBlockEntriesStmt *sql.Stmt
}
func createStateBlockTable(db *sql.DB) error {
func CreateStateBlockTable(db *sql.DB) error {
_, err := db.Exec(stateDataSchema)
return err
}
func prepareStateBlockTable(db *sql.DB) (tables.StateBlock, error) {
func PrepareStateBlockTable(db *sql.DB) (tables.StateBlock, error) {
s := &stateBlockStatements{}
return s, sqlutil.StatementList{
@ -90,9 +89,9 @@ func (s *stateBlockStatements) BulkInsertStateData(
entries types.StateEntries,
) (id types.StateBlockNID, err error) {
entries = entries[:util.SortAndUnique(entries)]
var nids types.EventNIDs
for _, e := range entries {
nids = append(nids, e.EventNID)
nids := make(types.EventNIDs, entries.Len())
for i := range entries {
nids[i] = entries[i].EventNID
}
stmt := sqlutil.TxStmt(txn, s.insertStateDataStmt)
err = stmt.QueryRowContext(
@ -113,15 +112,15 @@ func (s *stateBlockStatements) BulkSelectStateBlockEntries(
results := make([][]types.EventNID, len(stateBlockNIDs))
i := 0
var stateBlockNID types.StateBlockNID
var result pq.Int64Array
for ; rows.Next(); i++ {
var stateBlockNID types.StateBlockNID
var result pq.Int64Array
if err = rows.Scan(&stateBlockNID, &result); err != nil {
return nil, err
}
r := []types.EventNID{}
for _, e := range result {
r = append(r, types.EventNID(e))
r := make([]types.EventNID, len(result))
for x := range result {
r[x] = types.EventNID(result[x])
}
results[i] = r
}
@ -141,35 +140,3 @@ func stateBlockNIDsAsArray(stateBlockNIDs []types.StateBlockNID) pq.Int64Array {
}
return pq.Int64Array(nids)
}
type stateKeyTupleSorter []types.StateKeyTuple
func (s stateKeyTupleSorter) Len() int { return len(s) }
func (s stateKeyTupleSorter) Less(i, j int) bool { return s[i].LessThan(s[j]) }
func (s stateKeyTupleSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
// Check whether a tuple is in the list. Assumes that the list is sorted.
func (s stateKeyTupleSorter) contains(value types.StateKeyTuple) bool {
i := sort.Search(len(s), func(i int) bool { return !s[i].LessThan(value) })
return i < len(s) && s[i] == value
}
// List the unique eventTypeNIDs and eventStateKeyNIDs.
// Assumes that the list is sorted.
func (s stateKeyTupleSorter) typesAndStateKeysAsArrays() (eventTypeNIDs pq.Int64Array, eventStateKeyNIDs pq.Int64Array) {
eventTypeNIDs = make(pq.Int64Array, len(s))
eventStateKeyNIDs = make(pq.Int64Array, len(s))
for i := range s {
eventTypeNIDs[i] = int64(s[i].EventTypeNID)
eventStateKeyNIDs[i] = int64(s[i].EventStateKeyNID)
}
eventTypeNIDs = eventTypeNIDs[:util.SortAndUnique(int64Sorter(eventTypeNIDs))]
eventStateKeyNIDs = eventStateKeyNIDs[:util.SortAndUnique(int64Sorter(eventStateKeyNIDs))]
return
}
type int64Sorter []int64
func (s int64Sorter) Len() int { return len(s) }
func (s int64Sorter) Less(i, j int) bool { return s[i] < s[j] }
func (s int64Sorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] }

86
roomserver/storage/postgres/state_block_table_test.go
View file

@ -1,86 +0,0 @@
// Copyright 2017-2018 New Vector Ltd
// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package postgres
import (
"sort"
"testing"
"github.com/matrix-org/dendrite/roomserver/types"
)
func TestStateKeyTupleSorter(t *testing.T) {
input := stateKeyTupleSorter{
{EventTypeNID: 1, EventStateKeyNID: 2},
{EventTypeNID: 1, EventStateKeyNID: 4},
{EventTypeNID: 2, EventStateKeyNID: 2},
{EventTypeNID: 1, EventStateKeyNID: 1},
}
want := []types.StateKeyTuple{
{EventTypeNID: 1, EventStateKeyNID: 1},
{EventTypeNID: 1, EventStateKeyNID: 2},
{EventTypeNID: 1, EventStateKeyNID: 4},
{EventTypeNID: 2, EventStateKeyNID: 2},
}
doNotWant := []types.StateKeyTuple{
{EventTypeNID: 0, EventStateKeyNID: 0},
{EventTypeNID: 1, EventStateKeyNID: 3},
{EventTypeNID: 2, EventStateKeyNID: 1},
{EventTypeNID: 3, EventStateKeyNID: 1},
}
wantTypeNIDs := []int64{1, 2}
wantStateKeyNIDs := []int64{1, 2, 4}
// Sort the input and check it's in the right order.
sort.Sort(input)
gotTypeNIDs, gotStateKeyNIDs := input.typesAndStateKeysAsArrays()
for i := range want {
if input[i] != want[i] {
t.Errorf("Wanted %#v at index %d got %#v", want[i], i, input[i])
}
if !input.contains(want[i]) {
t.Errorf("Wanted %#v.contains(%#v) to be true but got false", input, want[i])
}
}
for i := range doNotWant {
if input.contains(doNotWant[i]) {
t.Errorf("Wanted %#v.contains(%#v) to be false but got true", input, doNotWant[i])
}
}
if len(wantTypeNIDs) != len(gotTypeNIDs) {
t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
}
for i := range wantTypeNIDs {
if wantTypeNIDs[i] != gotTypeNIDs[i] {
t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
}
}
if len(wantStateKeyNIDs) != len(gotStateKeyNIDs) {
t.Fatalf("Wanted state key NIDs %#v got %#v", wantStateKeyNIDs, gotStateKeyNIDs)
}
for i := range wantStateKeyNIDs {
if wantStateKeyNIDs[i] != gotStateKeyNIDs[i] {
t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
}
}
}

10
roomserver/storage/postgres/state_snapshot_table.go
View file

@ -77,12 +77,12 @@ type stateSnapshotStatements struct {
bulkSelectStateBlockNIDsStmt *sql.Stmt
}
func createStateSnapshotTable(db *sql.DB) error {
func CreateStateSnapshotTable(db *sql.DB) error {
_, err := db.Exec(stateSnapshotSchema)
return err
}
func prepareStateSnapshotTable(db *sql.DB) (tables.StateSnapshot, error) {
func PrepareStateSnapshotTable(db *sql.DB) (tables.StateSnapshot, error) {
s := &stateSnapshotStatements{}
return s, sqlutil.StatementList{
@ -95,12 +95,10 @@ func (s *stateSnapshotStatements) InsertState(
ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, nids types.StateBlockNIDs,
) (stateNID types.StateSnapshotNID, err error) {
nids = nids[:util.SortAndUnique(nids)]
var id int64
err = sqlutil.TxStmt(txn, s.insertStateStmt).QueryRowContext(ctx, nids.Hash(), int64(roomNID), stateBlockNIDsAsArray(nids)).Scan(&id)
err = sqlutil.TxStmt(txn, s.insertStateStmt).QueryRowContext(ctx, nids.Hash(), int64(roomNID), stateBlockNIDsAsArray(nids)).Scan(&stateNID)
if err != nil {
return 0, err
}
stateNID = types.StateSnapshotNID(id)
return
}
@ -119,9 +117,9 @@ func (s *stateSnapshotStatements) BulkSelectStateBlockNIDs(
defer rows.Close() // nolint: errcheck
results := make([]types.StateBlockNIDList, len(stateNIDs))
i := 0
var stateBlockNIDs pq.Int64Array
for ; rows.Next(); i++ {
result := &results[i]
var stateBlockNIDs pq.Int64Array
if err = rows.Scan(&result.StateSnapshotNID, &stateBlockNIDs); err != nil {
return nil, err
}

16
roomserver/storage/postgres/storage.go
View file

@ -80,19 +80,19 @@ func (d *Database) create(db *sql.DB) error {
if err := CreateEventsTable(db); err != nil {
return err
}
if err := createRoomsTable(db); err != nil {
if err := CreateRoomsTable(db); err != nil {
return err
}
if err := createStateBlockTable(db); err != nil {
if err := CreateStateBlockTable(db); err != nil {
return err
}
if err := createStateSnapshotTable(db); err != nil {
if err := CreateStateSnapshotTable(db); err != nil {
return err
}
if err := CreatePrevEventsTable(db); err != nil {
return err
}
if err := createRoomAliasesTable(db); err != nil {
if err := CreateRoomAliasesTable(db); err != nil {
return err
}
if err := CreateInvitesTable(db); err != nil {
@ -128,15 +128,15 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room
if err != nil {
return err
}
rooms, err := prepareRoomsTable(db)
rooms, err := PrepareRoomsTable(db)
if err != nil {
return err
}
stateBlock, err := prepareStateBlockTable(db)
stateBlock, err := PrepareStateBlockTable(db)
if err != nil {
return err
}
stateSnapshot, err := prepareStateSnapshotTable(db)
stateSnapshot, err := PrepareStateSnapshotTable(db)
if err != nil {
return err
}
@ -144,7 +144,7 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room
if err != nil {
return err
}
roomAliases, err := prepareRoomAliasesTable(db)
roomAliases, err := PrepareRoomAliasesTable(db)
if err != nil {
return err
}

2
roomserver/storage/shared/storage.go
View file

@ -1216,7 +1216,7 @@ func (d *Database) GetKnownUsers(ctx context.Context, userID, searchString strin
// GetKnownRooms returns a list of all rooms we know about.
func (d *Database) GetKnownRooms(ctx context.Context) ([]string, error) {
return d.RoomsTable.SelectRoomIDs(ctx, nil)
return d.RoomsTable.SelectRoomIDsWithEvents(ctx, nil)
}
// ForgetRoom sets a users room to forgotten

4
roomserver/storage/sqlite3/events_table.go
View file

@ -247,9 +247,9 @@ func (s *eventStatements) BulkSelectStateEventByNID(
ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID,
stateKeyTuples []types.StateKeyTuple,
) ([]types.StateEntry, error) {
tuples := stateKeyTupleSorter(stateKeyTuples)
tuples := types.StateKeyTupleSorter(stateKeyTuples)
sort.Sort(tuples)
eventTypeNIDArray, eventStateKeyNIDArray := tuples.typesAndStateKeysAsArrays()
eventTypeNIDArray, eventStateKeyNIDArray := tuples.TypesAndStateKeysAsArrays()
params := make([]interface{}, 0, len(eventNIDs)+len(eventTypeNIDArray)+len(eventStateKeyNIDArray))
selectOrig := strings.Replace(bulkSelectStateEventByNIDSQL, "($1)", sqlutil.QueryVariadic(len(eventNIDs)), 1)
for _, v := range eventNIDs {

6
roomserver/storage/sqlite3/room_aliases_table.go
View file

@ -63,12 +63,12 @@ type roomAliasesStatements struct {
deleteRoomAliasStmt *sql.Stmt
}
func createRoomAliasesTable(db *sql.DB) error {
func CreateRoomAliasesTable(db *sql.DB) error {
_, err := db.Exec(roomAliasesSchema)
return err
}
func prepareRoomAliasesTable(db *sql.DB) (tables.RoomAliases, error) {
func PrepareRoomAliasesTable(db *sql.DB) (tables.RoomAliases, error) {
s := &roomAliasesStatements{
db: db,
}
@ -113,8 +113,8 @@ func (s *roomAliasesStatements) SelectAliasesFromRoomID(
defer internal.CloseAndLogIfError(ctx, rows, "selectAliasesFromRoomID: rows.close() failed")
var alias string
for rows.Next() {
var alias string
if err = rows.Scan(&alias); err != nil {
return
}

16
roomserver/storage/sqlite3/rooms_table.go
View file

@ -86,12 +86,12 @@ type roomStatements struct {
selectRoomIDsStmt *sql.Stmt
}
func createRoomsTable(db *sql.DB) error {
func CreateRoomsTable(db *sql.DB) error {
_, err := db.Exec(roomsSchema)
return err
}
func prepareRoomsTable(db *sql.DB) (tables.Rooms, error) {
func PrepareRoomsTable(db *sql.DB) (tables.Rooms, error) {
s := &roomStatements{
db: db,
}
@ -108,7 +108,7 @@ func prepareRoomsTable(db *sql.DB) (tables.Rooms, error) {
}.Prepare(db)
}
func (s *roomStatements) SelectRoomIDs(ctx context.Context, txn *sql.Tx) ([]string, error) {
func (s *roomStatements) SelectRoomIDsWithEvents(ctx context.Context, txn *sql.Tx) ([]string, error) {
stmt := sqlutil.TxStmt(txn, s.selectRoomIDsStmt)
rows, err := stmt.QueryContext(ctx)
if err != nil {
@ -116,8 +116,8 @@ func (s *roomStatements) SelectRoomIDs(ctx context.Context, txn *sql.Tx) ([]stri
}
defer internal.CloseAndLogIfError(ctx, rows, "selectRoomIDsStmt: rows.close() failed")
var roomIDs []string
var roomID string
for rows.Next() {
var roomID string
if err = rows.Scan(&roomID); err != nil {
return nil, err
}
@ -241,9 +241,9 @@ func (s *roomStatements) SelectRoomVersionsForRoomNIDs(
}
defer internal.CloseAndLogIfError(ctx, rows, "selectRoomVersionsForRoomNIDsStmt: rows.close() failed")
result := make(map[types.RoomNID]gomatrixserverlib.RoomVersion)
var roomNID types.RoomNID
var roomVersion gomatrixserverlib.RoomVersion
for rows.Next() {
var roomNID types.RoomNID
var roomVersion gomatrixserverlib.RoomVersion
if err = rows.Scan(&roomNID, &roomVersion); err != nil {
return nil, err
}
@ -270,8 +270,8 @@ func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roo
}
defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectRoomIDsStmt: rows.close() failed")
var roomIDs []string
var roomID string
for rows.Next() {
var roomID string
if err = rows.Scan(&roomID); err != nil {
return nil, err
}
@ -298,8 +298,8 @@ func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, ro
}
defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectRoomNIDsStmt: rows.close() failed")
var roomNIDs []types.RoomNID
var roomNID types.RoomNID
for rows.Next() {
var roomNID types.RoomNID
if err = rows.Scan(&roomNID); err != nil {
return nil, err
}

49
roomserver/storage/sqlite3/state_block_table.go
View file

@ -20,7 +20,6 @@ import (
"database/sql"
"encoding/json"
"fmt"
"sort"
"strings"
"github.com/matrix-org/dendrite/internal"
@ -64,12 +63,12 @@ type stateBlockStatements struct {
bulkSelectStateBlockEntriesStmt *sql.Stmt
}
func createStateBlockTable(db *sql.DB) error {
func CreateStateBlockTable(db *sql.DB) error {
_, err := db.Exec(stateDataSchema)
return err
}
func prepareStateBlockTable(db *sql.DB) (tables.StateBlock, error) {
func PrepareStateBlockTable(db *sql.DB) (tables.StateBlock, error) {
s := &stateBlockStatements{
db: db,
}
@ -85,9 +84,9 @@ func (s *stateBlockStatements) BulkInsertStateData(
entries types.StateEntries,
) (id types.StateBlockNID, err error) {
entries = entries[:util.SortAndUnique(entries)]
nids := types.EventNIDs{} // zero slice to not store 'null' in the DB
for _, e := range entries {
nids = append(nids, e.EventNID)
nids := make(types.EventNIDs, entries.Len())
for i := range entries {
nids[i] = entries[i].EventNID
}
js, err := json.Marshal(nids)
if err != nil {
@ -122,13 +121,13 @@ func (s *stateBlockStatements) BulkSelectStateBlockEntries(
results := make([][]types.EventNID, len(stateBlockNIDs))
i := 0
var stateBlockNID types.StateBlockNID
var result json.RawMessage
for ; rows.Next(); i++ {
var stateBlockNID types.StateBlockNID
var result json.RawMessage
if err = rows.Scan(&stateBlockNID, &result); err != nil {
return nil, err
}
r := []types.EventNID{}
var r []types.EventNID
if err = json.Unmarshal(result, &r); err != nil {
return nil, fmt.Errorf("json.Unmarshal: %w", err)
}
@ -142,35 +141,3 @@ func (s *stateBlockStatements) BulkSelectStateBlockEntries(
}
return results, err
}
type stateKeyTupleSorter []types.StateKeyTuple
func (s stateKeyTupleSorter) Len() int { return len(s) }
func (s stateKeyTupleSorter) Less(i, j int) bool { return s[i].LessThan(s[j]) }
func (s stateKeyTupleSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
// Check whether a tuple is in the list. Assumes that the list is sorted.
func (s stateKeyTupleSorter) contains(value types.StateKeyTuple) bool {
i := sort.Search(len(s), func(i int) bool { return !s[i].LessThan(value) })
return i < len(s) && s[i] == value
}
// List the unique eventTypeNIDs and eventStateKeyNIDs.
// Assumes that the list is sorted.
func (s stateKeyTupleSorter) typesAndStateKeysAsArrays() (eventTypeNIDs []int64, eventStateKeyNIDs []int64) {
eventTypeNIDs = make([]int64, len(s))
eventStateKeyNIDs = make([]int64, len(s))
for i := range s {
eventTypeNIDs[i] = int64(s[i].EventTypeNID)
eventStateKeyNIDs[i] = int64(s[i].EventStateKeyNID)
}
eventTypeNIDs = eventTypeNIDs[:util.SortAndUnique(int64Sorter(eventTypeNIDs))]
eventStateKeyNIDs = eventStateKeyNIDs[:util.SortAndUnique(int64Sorter(eventStateKeyNIDs))]
return
}
type int64Sorter []int64
func (s int64Sorter) Len() int { return len(s) }
func (s int64Sorter) Less(i, j int) bool { return s[i] < s[j] }
func (s int64Sorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] }

86
roomserver/storage/sqlite3/state_block_table_test.go
View file

@ -1,86 +0,0 @@
// Copyright 2017-2018 New Vector Ltd
// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sqlite3
import (
"sort"
"testing"
"github.com/matrix-org/dendrite/roomserver/types"
)
func TestStateKeyTupleSorter(t *testing.T) {
input := stateKeyTupleSorter{
{EventTypeNID: 1, EventStateKeyNID: 2},
{EventTypeNID: 1, EventStateKeyNID: 4},
{EventTypeNID: 2, EventStateKeyNID: 2},
{EventTypeNID: 1, EventStateKeyNID: 1},
}
want := []types.StateKeyTuple{
{EventTypeNID: 1, EventStateKeyNID: 1},
{EventTypeNID: 1, EventStateKeyNID: 2},
{EventTypeNID: 1, EventStateKeyNID: 4},
{EventTypeNID: 2, EventStateKeyNID: 2},
}
doNotWant := []types.StateKeyTuple{
{EventTypeNID: 0, EventStateKeyNID: 0},
{EventTypeNID: 1, EventStateKeyNID: 3},
{EventTypeNID: 2, EventStateKeyNID: 1},
{EventTypeNID: 3, EventStateKeyNID: 1},
}
wantTypeNIDs := []int64{1, 2}
wantStateKeyNIDs := []int64{1, 2, 4}
// Sort the input and check it's in the right order.
sort.Sort(input)
gotTypeNIDs, gotStateKeyNIDs := input.typesAndStateKeysAsArrays()
for i := range want {
if input[i] != want[i] {
t.Errorf("Wanted %#v at index %d got %#v", want[i], i, input[i])
}
if !input.contains(want[i]) {
t.Errorf("Wanted %#v.contains(%#v) to be true but got false", input, want[i])
}
}
for i := range doNotWant {
if input.contains(doNotWant[i]) {
t.Errorf("Wanted %#v.contains(%#v) to be false but got true", input, doNotWant[i])
}
}
if len(wantTypeNIDs) != len(gotTypeNIDs) {
t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
}
for i := range wantTypeNIDs {
if wantTypeNIDs[i] != gotTypeNIDs[i] {
t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
}
}
if len(wantStateKeyNIDs) != len(gotStateKeyNIDs) {
t.Fatalf("Wanted state key NIDs %#v got %#v", wantStateKeyNIDs, gotStateKeyNIDs)
}
for i := range wantStateKeyNIDs {
if wantStateKeyNIDs[i] != gotStateKeyNIDs[i] {
t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
}
}
}

10
roomserver/storage/sqlite3/state_snapshot_table.go
View file

@ -68,12 +68,12 @@ type stateSnapshotStatements struct {
bulkSelectStateBlockNIDsStmt *sql.Stmt
}
func createStateSnapshotTable(db *sql.DB) error {
func CreateStateSnapshotTable(db *sql.DB) error {
_, err := db.Exec(stateSnapshotSchema)
return err
}
func prepareStateSnapshotTable(db *sql.DB) (tables.StateSnapshot, error) {
func PrepareStateSnapshotTable(db *sql.DB) (tables.StateSnapshot, error) {
s := &stateSnapshotStatements{
db: db,
}
@ -96,12 +96,10 @@ func (s *stateSnapshotStatements) InsertState(
return
}
insertStmt := sqlutil.TxStmt(txn, s.insertStateStmt)
var id int64
err = insertStmt.QueryRowContext(ctx, stateBlockNIDs.Hash(), int64(roomNID), string(stateBlockNIDsJSON)).Scan(&id)
err = insertStmt.QueryRowContext(ctx, stateBlockNIDs.Hash(), int64(roomNID), string(stateBlockNIDsJSON)).Scan(&stateNID)
if err != nil {
return 0, err
}
stateNID = types.StateSnapshotNID(id)
return
}
@ -127,9 +125,9 @@ func (s *stateSnapshotStatements) BulkSelectStateBlockNIDs(
defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectStateBlockNIDs: rows.close() failed")
results := make([]types.StateBlockNIDList, len(stateNIDs))
i := 0
var stateBlockNIDsJSON string
for ; rows.Next(); i++ {
result := &results[i]
var stateBlockNIDsJSON string
if err := rows.Scan(&result.StateSnapshotNID, &stateBlockNIDsJSON); err != nil {
return nil, err
}

16
roomserver/storage/sqlite3/storage.go
View file

@ -89,19 +89,19 @@ func (d *Database) create(db *sql.DB) error {
if err := CreateEventsTable(db); err != nil {
return err
}
if err := createRoomsTable(db); err != nil {
if err := CreateRoomsTable(db); err != nil {
return err
}
if err := createStateBlockTable(db); err != nil {
if err := CreateStateBlockTable(db); err != nil {
return err
}
if err := createStateSnapshotTable(db); err != nil {
if err := CreateStateSnapshotTable(db); err != nil {
return err
}
if err := CreatePrevEventsTable(db); err != nil {
return err
}
if err := createRoomAliasesTable(db); err != nil {
if err := CreateRoomAliasesTable(db); err != nil {
return err
}
if err := CreateInvitesTable(db); err != nil {
@ -137,15 +137,15 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room
if err != nil {
return err
}
rooms, err := prepareRoomsTable(db)
rooms, err := PrepareRoomsTable(db)
if err != nil {
return err
}
stateBlock, err := prepareStateBlockTable(db)
stateBlock, err := PrepareStateBlockTable(db)
if err != nil {
return err
}
stateSnapshot, err := prepareStateSnapshotTable(db)
stateSnapshot, err := PrepareStateSnapshotTable(db)
if err != nil {
return err
}
@ -153,7 +153,7 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room
if err != nil {
return err
}
roomAliases, err := prepareRoomAliasesTable(db)
roomAliases, err := PrepareRoomAliasesTable(db)
if err != nil {
return err
}

3
roomserver/storage/tables/events_table_test.go
View file

@ -39,7 +39,7 @@ func mustCreateEventsTable(t *testing.T, dbType test.DBType) (tables.Events, fun
}
func Test_EventsTable(t *testing.T) {
alice := test.NewUser()
alice := test.NewUser(t)
room := test.NewRoom(t, alice)
ctx := context.Background()
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
@ -85,7 +85,6 @@ func Test_EventsTable(t *testing.T) {
assert.NoError(t, err)
}
stateAtEvent := types.StateAtEvent{
Overwrite: false,
BeforeStateSnapshotNID: types.StateSnapshotNID(stateSnapshot),
IsRejected: false,
StateEntry: types.StateEntry{

2
roomserver/storage/tables/interface.go
View file

@ -72,7 +72,7 @@ type Rooms interface {
UpdateLatestEventNIDs(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, eventNIDs []types.EventNID, lastEventSentNID types.EventNID, stateSnapshotNID types.StateSnapshotNID) error
SelectRoomVersionsForRoomNIDs(ctx context.Context, txn *sql.Tx, roomNID []types.RoomNID) (map[types.RoomNID]gomatrixserverlib.RoomVersion, error)
SelectRoomInfo(ctx context.Context, txn *sql.Tx, roomID string) (*types.RoomInfo, error)
SelectRoomIDs(ctx context.Context, txn *sql.Tx) ([]string, error)
SelectRoomIDsWithEvents(ctx context.Context, txn *sql.Tx) ([]string, error)
BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID) ([]string, error)
BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, roomIDs []string) ([]types.RoomNID, error)
}

2
roomserver/storage/tables/previous_events_table_test.go
View file

@ -38,7 +38,7 @@ func mustCreatePreviousEventsTable(t *testing.T, dbType test.DBType) (tab tables
func TestPreviousEventsTable(t *testing.T) {
ctx := context.Background()
alice := test.NewUser()
alice := test.NewUser(t)
room := test.NewRoom(t, alice)
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
tab, close := mustCreatePreviousEventsTable(t, dbType)

2
roomserver/storage/tables/published_table_test.go
View file

@ -38,7 +38,7 @@ func mustCreatePublishedTable(t *testing.T, dbType test.DBType) (tab tables.Publ
func TestPublishedTable(t *testing.T) {
ctx := context.Background()
alice := test.NewUser()
alice := test.NewUser(t)
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
tab, close := mustCreatePublishedTable(t, dbType)

96
roomserver/storage/tables/room_aliases_table_test.go Normal file
View file

@ -0,0 +1,96 @@
package tables_test
import (
"context"
"testing"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/roomserver/storage/postgres"
"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
"github.com/matrix-org/dendrite/roomserver/storage/tables"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/test"
"github.com/stretchr/testify/assert"
)
func mustCreateRoomAliasesTable(t *testing.T, dbType test.DBType) (tab tables.RoomAliases, close func()) {
t.Helper()
connStr, close := test.PrepareDBConnectionString(t, dbType)
db, err := sqlutil.Open(&config.DatabaseOptions{
ConnectionString: config.DataSource(connStr),
}, sqlutil.NewExclusiveWriter())
assert.NoError(t, err)
switch dbType {
case test.DBTypePostgres:
err = postgres.CreateRoomAliasesTable(db)
assert.NoError(t, err)
tab, err = postgres.PrepareRoomAliasesTable(db)
case test.DBTypeSQLite:
err = sqlite3.CreateRoomAliasesTable(db)
assert.NoError(t, err)
tab, err = sqlite3.PrepareRoomAliasesTable(db)
}
assert.NoError(t, err)
return tab, close
}
func TestRoomAliasesTable(t *testing.T) {
alice := test.NewUser(t)
room := test.NewRoom(t, alice)
room2 := test.NewRoom(t, alice)
ctx := context.Background()
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
tab, close := mustCreateRoomAliasesTable(t, dbType)
defer close()
alias, alias2, alias3 := "#alias:localhost", "#alias2:localhost", "#alias3:localhost"
// insert aliases
err := tab.InsertRoomAlias(ctx, nil, alias, room.ID, alice.ID)
assert.NoError(t, err)
err = tab.InsertRoomAlias(ctx, nil, alias2, room.ID, alice.ID)
assert.NoError(t, err)
err = tab.InsertRoomAlias(ctx, nil, alias3, room2.ID, alice.ID)
assert.NoError(t, err)
// verify we can get the roomID for the alias
roomID, err := tab.SelectRoomIDFromAlias(ctx, nil, alias)
assert.NoError(t, err)
assert.Equal(t, room.ID, roomID)
// .. and the creator
creator, err := tab.SelectCreatorIDFromAlias(ctx, nil, alias)
assert.NoError(t, err)
assert.Equal(t, alice.ID, creator)
creator, err = tab.SelectCreatorIDFromAlias(ctx, nil, "#doesntexist:localhost")
assert.NoError(t, err)
assert.Equal(t, "", creator)
roomID, err = tab.SelectRoomIDFromAlias(ctx, nil, "#doesntexist:localhost")
assert.NoError(t, err)
assert.Equal(t, "", roomID)
// get all aliases for a room
aliases, err := tab.SelectAliasesFromRoomID(ctx, nil, room.ID)
assert.NoError(t, err)
assert.Equal(t, []string{alias, alias2}, aliases)
// delete an alias and verify it's deleted
err = tab.DeleteRoomAlias(ctx, nil, alias2)
assert.NoError(t, err)
aliases, err = tab.SelectAliasesFromRoomID(ctx, nil, room.ID)
assert.NoError(t, err)
assert.Equal(t, []string{alias}, aliases)
// deleting the same alias should be a no-op
err = tab.DeleteRoomAlias(ctx, nil, alias2)
assert.NoError(t, err)
// Delete non-existent alias should be a no-op
err = tab.DeleteRoomAlias(ctx, nil, "#doesntexist:localhost")
assert.NoError(t, err)
})
}

128
roomserver/storage/tables/rooms_table_test.go Normal file
View file

@ -0,0 +1,128 @@
package tables_test
import (
"context"
"testing"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/roomserver/storage/postgres"
"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
"github.com/matrix-org/dendrite/roomserver/storage/tables"
"github.com/matrix-org/dendrite/roomserver/types"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/util"
"github.com/stretchr/testify/assert"
)
func mustCreateRoomsTable(t *testing.T, dbType test.DBType) (tab tables.Rooms, close func()) {
t.Helper()
connStr, close := test.PrepareDBConnectionString(t, dbType)
db, err := sqlutil.Open(&config.DatabaseOptions{
ConnectionString: config.DataSource(connStr),
}, sqlutil.NewExclusiveWriter())
assert.NoError(t, err)
switch dbType {
case test.DBTypePostgres:
err = postgres.CreateRoomsTable(db)
assert.NoError(t, err)
tab, err = postgres.PrepareRoomsTable(db)
case test.DBTypeSQLite:
err = sqlite3.CreateRoomsTable(db)
assert.NoError(t, err)
tab, err = sqlite3.PrepareRoomsTable(db)
}
assert.NoError(t, err)
return tab, close
}
func TestRoomsTable(t *testing.T) {
alice := test.NewUser(t)
room := test.NewRoom(t, alice)
ctx := context.Background()
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
tab, close := mustCreateRoomsTable(t, dbType)
defer close()
wantRoomNID, err := tab.InsertRoomNID(ctx, nil, room.ID, room.Version)
assert.NoError(t, err)
// Create dummy room
_, err = tab.InsertRoomNID(ctx, nil, util.RandomString(16), room.Version)
assert.NoError(t, err)
gotRoomNID, err := tab.SelectRoomNID(ctx, nil, room.ID)
assert.NoError(t, err)
assert.Equal(t, wantRoomNID, gotRoomNID)
// Ensure non existent roomNID errors
roomNID, err := tab.SelectRoomNID(ctx, nil, "!doesnotexist:localhost")
assert.Error(t, err)
assert.Equal(t, types.RoomNID(0), roomNID)
roomInfo, err := tab.SelectRoomInfo(ctx, nil, room.ID)
assert.NoError(t, err)
assert.Equal(t, &types.RoomInfo{
RoomNID: wantRoomNID,
RoomVersion: room.Version,
StateSnapshotNID: 0,
IsStub: true, // there are no latestEventNIDs
}, roomInfo)
roomInfo, err = tab.SelectRoomInfo(ctx, nil, "!doesnotexist:localhost")
assert.NoError(t, err)
assert.Nil(t, roomInfo)
// There are no rooms with latestEventNIDs yet
roomIDs, err := tab.SelectRoomIDsWithEvents(ctx, nil)
assert.NoError(t, err)
assert.Equal(t, 0, len(roomIDs))
roomVersions, err := tab.SelectRoomVersionsForRoomNIDs(ctx, nil, []types.RoomNID{wantRoomNID, 1337})
assert.NoError(t, err)
assert.Equal(t, roomVersions[wantRoomNID], room.Version)
// Room does not exist
_, ok := roomVersions[1337]
assert.False(t, ok)
roomIDs, err = tab.BulkSelectRoomIDs(ctx, nil, []types.RoomNID{wantRoomNID, 1337})
assert.NoError(t, err)
assert.Equal(t, []string{room.ID}, roomIDs)
roomNIDs, err := tab.BulkSelectRoomNIDs(ctx, nil, []string{room.ID, "!doesnotexist:localhost"})
assert.NoError(t, err)
assert.Equal(t, []types.RoomNID{wantRoomNID}, roomNIDs)
wantEventNIDs := []types.EventNID{1, 2, 3}
lastEventSentNID := types.EventNID(3)
stateSnapshotNID := types.StateSnapshotNID(1)
// make the room "usable"
err = tab.UpdateLatestEventNIDs(ctx, nil, wantRoomNID, wantEventNIDs, lastEventSentNID, stateSnapshotNID)
assert.NoError(t, err)
roomInfo, err = tab.SelectRoomInfo(ctx, nil, room.ID)
assert.NoError(t, err)
assert.Equal(t, &types.RoomInfo{
RoomNID: wantRoomNID,
RoomVersion: room.Version,
StateSnapshotNID: 1,
IsStub: false,
}, roomInfo)
eventNIDs, snapshotNID, err := tab.SelectLatestEventNIDs(ctx, nil, wantRoomNID)
assert.NoError(t, err)
assert.Equal(t, wantEventNIDs, eventNIDs)
assert.Equal(t, types.StateSnapshotNID(1), snapshotNID)
// Again, doesn't exist
_, _, err = tab.SelectLatestEventNIDs(ctx, nil, 1337)
assert.Error(t, err)
eventNIDs, eventNID, snapshotNID, err := tab.SelectLatestEventsNIDsForUpdate(ctx, nil, wantRoomNID)
assert.NoError(t, err)
assert.Equal(t, wantEventNIDs, eventNIDs)
assert.Equal(t, types.EventNID(3), eventNID)
assert.Equal(t, types.StateSnapshotNID(1), snapshotNID)
})
}

92
roomserver/storage/tables/state_block_table_test.go Normal file
View file

@ -0,0 +1,92 @@
package tables_test
import (
"context"
"testing"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/roomserver/storage/postgres"
"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
"github.com/matrix-org/dendrite/roomserver/storage/tables"
"github.com/matrix-org/dendrite/roomserver/types"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/test"
"github.com/stretchr/testify/assert"
)
func mustCreateStateBlockTable(t *testing.T, dbType test.DBType) (tab tables.StateBlock, close func()) {
t.Helper()
connStr, close := test.PrepareDBConnectionString(t, dbType)
db, err := sqlutil.Open(&config.DatabaseOptions{
ConnectionString: config.DataSource(connStr),
}, sqlutil.NewExclusiveWriter())
assert.NoError(t, err)
switch dbType {
case test.DBTypePostgres:
err = postgres.CreateStateBlockTable(db)
assert.NoError(t, err)
tab, err = postgres.PrepareStateBlockTable(db)
case test.DBTypeSQLite:
err = sqlite3.CreateStateBlockTable(db)
assert.NoError(t, err)
tab, err = sqlite3.PrepareStateBlockTable(db)
}
assert.NoError(t, err)
return tab, close
}
func TestStateBlockTable(t *testing.T) {
ctx := context.Background()
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
tab, close := mustCreateStateBlockTable(t, dbType)
defer close()
// generate some dummy data
var entries types.StateEntries
for i := 0; i < 100; i++ {
entry := types.StateEntry{
EventNID: types.EventNID(i),
}
entries = append(entries, entry)
}
stateBlockNID, err := tab.BulkInsertStateData(ctx, nil, entries)
assert.NoError(t, err)
assert.Equal(t, types.StateBlockNID(1), stateBlockNID)
// generate a different hash, to get a new StateBlockNID
var entries2 types.StateEntries
for i := 100; i < 300; i++ {
entry := types.StateEntry{
EventNID: types.EventNID(i),
}
entries2 = append(entries2, entry)
}
stateBlockNID, err = tab.BulkInsertStateData(ctx, nil, entries2)
assert.NoError(t, err)
assert.Equal(t, types.StateBlockNID(2), stateBlockNID)
eventNIDs, err := tab.BulkSelectStateBlockEntries(ctx, nil, types.StateBlockNIDs{1, 2})
assert.NoError(t, err)
assert.Equal(t, len(entries), len(eventNIDs[0]))
assert.Equal(t, len(entries2), len(eventNIDs[1]))
// try to get a StateBlockNID which does not exist
_, err = tab.BulkSelectStateBlockEntries(ctx, nil, types.StateBlockNIDs{5})
assert.Error(t, err)
// This should return an error, since we can only retrieve 1 StateBlock
_, err = tab.BulkSelectStateBlockEntries(ctx, nil, types.StateBlockNIDs{1, 5})
assert.Error(t, err)
for i := 0; i < 65555; i++ {
entry := types.StateEntry{
EventNID: types.EventNID(i),
}
entries2 = append(entries2, entry)
}
stateBlockNID, err = tab.BulkInsertStateData(ctx, nil, entries2)
assert.NoError(t, err)
assert.Equal(t, types.StateBlockNID(3), stateBlockNID)
})
}

86
roomserver/storage/tables/state_snapshot_table_test.go Normal file
View file

@ -0,0 +1,86 @@
package tables_test
import (
"context"
"testing"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/roomserver/storage/postgres"
"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
"github.com/matrix-org/dendrite/roomserver/storage/tables"
"github.com/matrix-org/dendrite/roomserver/types"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/test"
"github.com/stretchr/testify/assert"
)
func mustCreateStateSnapshotTable(t *testing.T, dbType test.DBType) (tab tables.StateSnapshot, close func()) {
t.Helper()
connStr, close := test.PrepareDBConnectionString(t, dbType)
db, err := sqlutil.Open(&config.DatabaseOptions{
ConnectionString: config.DataSource(connStr),
}, sqlutil.NewExclusiveWriter())
assert.NoError(t, err)
switch dbType {
case test.DBTypePostgres:
err = postgres.CreateStateSnapshotTable(db)
assert.NoError(t, err)
tab, err = postgres.PrepareStateSnapshotTable(db)
case test.DBTypeSQLite:
err = sqlite3.CreateStateSnapshotTable(db)
assert.NoError(t, err)
tab, err = sqlite3.PrepareStateSnapshotTable(db)
}
assert.NoError(t, err)
return tab, close
}
func TestStateSnapshotTable(t *testing.T) {
ctx := context.Background()
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
tab, close := mustCreateStateSnapshotTable(t, dbType)
defer close()
// generate some dummy data
var stateBlockNIDs types.StateBlockNIDs
for i := 0; i < 100; i++ {
stateBlockNIDs = append(stateBlockNIDs, types.StateBlockNID(i))
}
stateNID, err := tab.InsertState(ctx, nil, 1, stateBlockNIDs)
assert.NoError(t, err)
assert.Equal(t, types.StateSnapshotNID(1), stateNID)
// verify ON CONFLICT; Note: this updates the sequence!
stateNID, err = tab.InsertState(ctx, nil, 1, stateBlockNIDs)
assert.NoError(t, err)
assert.Equal(t, types.StateSnapshotNID(1), stateNID)
// create a second snapshot
var stateBlockNIDs2 types.StateBlockNIDs
for i := 100; i < 150; i++ {
stateBlockNIDs2 = append(stateBlockNIDs2, types.StateBlockNID(i))
}
stateNID, err = tab.InsertState(ctx, nil, 1, stateBlockNIDs2)
assert.NoError(t, err)
// StateSnapshotNID is now 3, since the DO UPDATE SET statement incremented the sequence
assert.Equal(t, types.StateSnapshotNID(3), stateNID)
nidLists, err := tab.BulkSelectStateBlockNIDs(ctx, nil, []types.StateSnapshotNID{1, 3})
assert.NoError(t, err)
assert.Equal(t, stateBlockNIDs, types.StateBlockNIDs(nidLists[0].StateBlockNIDs))
assert.Equal(t, stateBlockNIDs2, types.StateBlockNIDs(nidLists[1].StateBlockNIDs))
// check we get an error if the state snapshot does not exist
_, err = tab.BulkSelectStateBlockNIDs(ctx, nil, []types.StateSnapshotNID{2})
assert.Error(t, err)
// create a second snapshot
for i := 0; i < 65555; i++ {
stateBlockNIDs2 = append(stateBlockNIDs2, types.StateBlockNID(i))
}
_, err = tab.InsertState(ctx, nil, 1, stateBlockNIDs2)
assert.NoError(t, err)
})
}

60
roomserver/types/types.go
View file

@ -18,8 +18,10 @@ package types
import (
"encoding/json"
"sort"
"strings"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"golang.org/x/crypto/blake2b"
)
@ -96,6 +98,38 @@ func (a StateKeyTuple) LessThan(b StateKeyTuple) bool {
return a.EventStateKeyNID < b.EventStateKeyNID
}
type StateKeyTupleSorter []StateKeyTuple
func (s StateKeyTupleSorter) Len() int { return len(s) }
func (s StateKeyTupleSorter) Less(i, j int) bool { return s[i].LessThan(s[j]) }
func (s StateKeyTupleSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
// Check whether a tuple is in the list. Assumes that the list is sorted.
func (s StateKeyTupleSorter) contains(value StateKeyTuple) bool {
i := sort.Search(len(s), func(i int) bool { return !s[i].LessThan(value) })
return i < len(s) && s[i] == value
}
// List the unique eventTypeNIDs and eventStateKeyNIDs.
// Assumes that the list is sorted.
func (s StateKeyTupleSorter) TypesAndStateKeysAsArrays() (eventTypeNIDs []int64, eventStateKeyNIDs []int64) {
eventTypeNIDs = make([]int64, len(s))
eventStateKeyNIDs = make([]int64, len(s))
for i := range s {
eventTypeNIDs[i] = int64(s[i].EventTypeNID)
eventStateKeyNIDs[i] = int64(s[i].EventStateKeyNID)
}
eventTypeNIDs = eventTypeNIDs[:util.SortAndUnique(int64Sorter(eventTypeNIDs))]
eventStateKeyNIDs = eventStateKeyNIDs[:util.SortAndUnique(int64Sorter(eventStateKeyNIDs))]
return
}
type int64Sorter []int64
func (s int64Sorter) Len() int { return len(s) }
func (s int64Sorter) Less(i, j int) bool { return s[i] < s[j] }
func (s int64Sorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
// A StateEntry is an entry in the room state of a matrix room.
type StateEntry struct {
StateKeyTuple
@ -139,10 +173,6 @@ func DeduplicateStateEntries(a []StateEntry) []StateEntry {
// StateAtEvent is the state before and after a matrix event.
type StateAtEvent struct {
// Should this state overwrite the latest events and memberships of the room?
// This might be necessary when rejoining a federated room after a period of
// absence, as our state and latest events will be out of date.
Overwrite bool
// The state before the event.
BeforeStateSnapshotNID StateSnapshotNID
// True if this StateEntry is rejected. State resolution should then treat this
@ -166,6 +196,28 @@ type StateAtEventAndReference struct {
gomatrixserverlib.EventReference
}
type StateAtEventAndReferences []StateAtEventAndReference
func (s StateAtEventAndReferences) Less(a, b int) bool {
return strings.Compare(s[a].EventID, s[b].EventID) < 0
}
func (s StateAtEventAndReferences) Len() int {
return len(s)
}
func (s StateAtEventAndReferences) Swap(a, b int) {
s[a], s[b] = s[b], s[a]
}
func (s StateAtEventAndReferences) EventIDs() string {
strs := make([]string, 0, len(s))
for _, r := range s {
strs = append(strs, r.EventID)
}
return "[" + strings.Join(strs, " ") + "]"
}
// An Event is a gomatrixserverlib.Event with the numeric event ID attached.
// It is when performing bulk event lookup in the database.
type Event struct {

64
roomserver/types/types_test.go
View file

@ -1,6 +1,7 @@
package types
import (
"sort"
"testing"
)
@ -24,3 +25,66 @@ func TestDeduplicateStateEntries(t *testing.T) {
}
}
}
func TestStateKeyTupleSorter(t *testing.T) {
input := StateKeyTupleSorter{
{EventTypeNID: 1, EventStateKeyNID: 2},
{EventTypeNID: 1, EventStateKeyNID: 4},
{EventTypeNID: 2, EventStateKeyNID: 2},
{EventTypeNID: 1, EventStateKeyNID: 1},
}
want := []StateKeyTuple{
{EventTypeNID: 1, EventStateKeyNID: 1},
{EventTypeNID: 1, EventStateKeyNID: 2},
{EventTypeNID: 1, EventStateKeyNID: 4},
{EventTypeNID: 2, EventStateKeyNID: 2},
}
doNotWant := []StateKeyTuple{
{EventTypeNID: 0, EventStateKeyNID: 0},
{EventTypeNID: 1, EventStateKeyNID: 3},
{EventTypeNID: 2, EventStateKeyNID: 1},
{EventTypeNID: 3, EventStateKeyNID: 1},
}
wantTypeNIDs := []int64{1, 2}
wantStateKeyNIDs := []int64{1, 2, 4}
// Sort the input and check it's in the right order.
sort.Sort(input)
gotTypeNIDs, gotStateKeyNIDs := input.TypesAndStateKeysAsArrays()
for i := range want {
if input[i] != want[i] {
t.Errorf("Wanted %#v at index %d got %#v", want[i], i, input[i])
}
if !input.contains(want[i]) {
t.Errorf("Wanted %#v.contains(%#v) to be true but got false", input, want[i])
}
}
for i := range doNotWant {
if input.contains(doNotWant[i]) {
t.Errorf("Wanted %#v.contains(%#v) to be false but got true", input, doNotWant[i])
}
}
if len(wantTypeNIDs) != len(gotTypeNIDs) {
t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
}
for i := range wantTypeNIDs {
if wantTypeNIDs[i] != gotTypeNIDs[i] {
t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
}
}
if len(wantStateKeyNIDs) != len(gotStateKeyNIDs) {
t.Fatalf("Wanted state key NIDs %#v got %#v", wantStateKeyNIDs, gotStateKeyNIDs)
}
for i := range wantStateKeyNIDs {
if wantStateKeyNIDs[i] != gotStateKeyNIDs[i] {
t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
}
}
}

7
setup/config/config_appservice.go
View file

@ -50,11 +50,14 @@ func (c *AppServiceAPI) Defaults(generate bool) {
}
func (c *AppServiceAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkURL(configErrs, "app_service_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "app_service_api.internal_api.bind", string(c.InternalAPI.Connect))
if c.Matrix.DatabaseOptions.ConnectionString == "" {
checkNotEmpty(configErrs, "app_service_api.database.connection_string", string(c.Database.ConnectionString))
}
if isMonolith { // polylith required configs below
return
}
checkURL(configErrs, "app_service_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "app_service_api.internal_api.connect", string(c.InternalAPI.Connect))
}
// ApplicationServiceNamespace is the namespace that a specific application

22
setup/config/config_clientapi.go
View file

@ -75,19 +75,13 @@ func (c *ClientAPI) Defaults(generate bool) {
}
func (c *ClientAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkURL(configErrs, "client_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "client_api.internal_api.connect", string(c.InternalAPI.Connect))
if !isMonolith {
checkURL(configErrs, "client_api.external_api.listen", string(c.ExternalAPI.Listen))
}
if c.RecaptchaEnabled {
checkNotEmpty(configErrs, "client_api.recaptcha_public_key", string(c.RecaptchaPublicKey))
checkNotEmpty(configErrs, "client_api.recaptcha_private_key", string(c.RecaptchaPrivateKey))
checkNotEmpty(configErrs, "client_api.recaptcha_siteverify_api", string(c.RecaptchaSiteVerifyAPI))
}
c.TURN.Verify(configErrs)
c.RateLimiting.Verify(configErrs)
if c.RecaptchaEnabled {
checkNotEmpty(configErrs, "client_api.recaptcha_public_key", c.RecaptchaPublicKey)
checkNotEmpty(configErrs, "client_api.recaptcha_private_key", c.RecaptchaPrivateKey)
checkNotEmpty(configErrs, "client_api.recaptcha_siteverify_api", c.RecaptchaSiteVerifyAPI)
}
// Ensure there is any spam counter measure when enabling registration
if !c.RegistrationDisabled && !c.OpenRegistrationWithoutVerificationEnabled {
if !c.RecaptchaEnabled {
@ -101,6 +95,12 @@ func (c *ClientAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
)
}
}
if isMonolith { // polylith required configs below
return
}
checkURL(configErrs, "client_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "client_api.internal_api.connect", string(c.InternalAPI.Connect))
checkURL(configErrs, "client_api.external_api.listen", string(c.ExternalAPI.Listen))
}
type TURN struct {

16
setup/config/config_federationapi.go
View file

@ -34,24 +34,24 @@ func (c *FederationAPI) Defaults(generate bool) {
c.InternalAPI.Listen = "http://localhost:7772"
c.InternalAPI.Connect = "http://localhost:7772"
c.ExternalAPI.Listen = "http://[::]:8072"
c.FederationMaxRetries = 16
c.DisableTLSValidation = false
c.Database.Defaults(10)
if generate {
c.Database.ConnectionString = "file:federationapi.db"
}
c.FederationMaxRetries = 16
c.DisableTLSValidation = false
}
func (c *FederationAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkURL(configErrs, "federation_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "federation_api.internal_api.connect", string(c.InternalAPI.Connect))
if !isMonolith {
checkURL(configErrs, "federation_api.external_api.listen", string(c.ExternalAPI.Listen))
}
if c.Matrix.DatabaseOptions.ConnectionString == "" {
checkNotEmpty(configErrs, "federation_api.database.connection_string", string(c.Database.ConnectionString))
}
if isMonolith { // polylith required configs below
return
}
checkURL(configErrs, "federation_api.external_api.listen", string(c.ExternalAPI.Listen))
checkURL(configErrs, "federation_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "federation_api.internal_api.connect", string(c.InternalAPI.Connect))
}
// The config for setting a proxy to use for server->server requests

7
setup/config/config_jetstream.go
View file

@ -36,9 +36,10 @@ func (c *JetStream) Defaults(generate bool) {
}
func (c *JetStream) Verify(configErrs *ConfigErrors, isMonolith bool) {
if isMonolith { // polylith required configs below
return
}
// If we are running in a polylith deployment then we need at least
// one NATS JetStream server to talk to.
if !isMonolith {
checkNotZero(configErrs, "global.jetstream.addresses", int64(len(c.Addresses)))
}
checkNotZero(configErrs, "global.jetstream.addresses", int64(len(c.Addresses)))
}

7
setup/config/config_keyserver.go
View file

@ -18,9 +18,12 @@ func (c *KeyServer) Defaults(generate bool) {
}
func (c *KeyServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkURL(configErrs, "key_server.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "key_server.internal_api.bind", string(c.InternalAPI.Connect))
if c.Matrix.DatabaseOptions.ConnectionString == "" {
checkNotEmpty(configErrs, "key_server.database.connection_string", string(c.Database.ConnectionString))
}
if isMonolith { // polylith required configs below
return
}
checkURL(configErrs, "key_server.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "key_server.internal_api.connect", string(c.InternalAPI.Connect))
}

17
setup/config/config_mediaapi.go
View file

@ -42,26 +42,19 @@ func (c *MediaAPI) Defaults(generate bool) {
c.InternalAPI.Listen = "http://localhost:7774"
c.InternalAPI.Connect = "http://localhost:7774"
c.ExternalAPI.Listen = "http://[::]:8074"
c.MaxFileSizeBytes = DefaultMaxFileSizeBytes
c.MaxThumbnailGenerators = 10
c.Database.Defaults(5)
if generate {
c.Database.ConnectionString = "file:mediaapi.db"
c.BasePath = "./media_store"
}
c.MaxFileSizeBytes = DefaultMaxFileSizeBytes
c.MaxThumbnailGenerators = 10
}
func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkURL(configErrs, "media_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "media_api.internal_api.connect", string(c.InternalAPI.Connect))
if !isMonolith {
checkURL(configErrs, "media_api.external_api.listen", string(c.ExternalAPI.Listen))
}
if c.Matrix.DatabaseOptions.ConnectionString == "" {
checkNotEmpty(configErrs, "media_api.database.connection_string", string(c.Database.ConnectionString))
}
checkNotEmpty(configErrs, "media_api.base_path", string(c.BasePath))
checkPositive(configErrs, "media_api.max_file_size_bytes", int64(c.MaxFileSizeBytes))
checkPositive(configErrs, "media_api.max_thumbnail_generators", int64(c.MaxThumbnailGenerators))
@ -70,4 +63,10 @@ func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkPositive(configErrs, fmt.Sprintf("media_api.thumbnail_sizes[%d].width", i), int64(size.Width))
checkPositive(configErrs, fmt.Sprintf("media_api.thumbnail_sizes[%d].height", i), int64(size.Height))
}
if isMonolith { // polylith required configs below
return
}
checkURL(configErrs, "media_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "media_api.internal_api.connect", string(c.InternalAPI.Connect))
checkURL(configErrs, "media_api.external_api.listen", string(c.ExternalAPI.Listen))
}

7
setup/config/config_roomserver.go
View file

@ -18,9 +18,12 @@ func (c *RoomServer) Defaults(generate bool) {
}
func (c *RoomServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkURL(configErrs, "room_server.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "room_server.internal_ap.bind", string(c.InternalAPI.Connect))
if c.Matrix.DatabaseOptions.ConnectionString == "" {
checkNotEmpty(configErrs, "room_server.database.connection_string", string(c.Database.ConnectionString))
}
if isMonolith { // polylith required configs below
return
}
checkURL(configErrs, "room_server.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "room_server.internal_ap.connect", string(c.InternalAPI.Connect))
}

11
setup/config/config_syncapi.go
View file

@ -22,12 +22,13 @@ func (c *SyncAPI) Defaults(generate bool) {
}
func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkURL(configErrs, "sync_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "sync_api.internal_api.bind", string(c.InternalAPI.Connect))
if !isMonolith {
checkURL(configErrs, "sync_api.external_api.listen", string(c.ExternalAPI.Listen))
}
if c.Matrix.DatabaseOptions.ConnectionString == "" {
checkNotEmpty(configErrs, "sync_api.database", string(c.Database.ConnectionString))
}
if isMonolith { // polylith required configs below
return
}
checkURL(configErrs, "sync_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "sync_api.internal_api.connect", string(c.InternalAPI.Connect))
checkURL(configErrs, "sync_api.external_api.listen", string(c.ExternalAPI.Listen))
}

13
setup/config/config_userapi.go
View file

@ -26,19 +26,22 @@ const DefaultOpenIDTokenLifetimeMS = 3600000 // 60 minutes
func (c *UserAPI) Defaults(generate bool) {
c.InternalAPI.Listen = "http://localhost:7781"
c.InternalAPI.Connect = "http://localhost:7781"
c.BCryptCost = bcrypt.DefaultCost
c.OpenIDTokenLifetimeMS = DefaultOpenIDTokenLifetimeMS
c.AccountDatabase.Defaults(10)
if generate {
c.AccountDatabase.ConnectionString = "file:userapi_accounts.db"
}
c.BCryptCost = bcrypt.DefaultCost
c.OpenIDTokenLifetimeMS = DefaultOpenIDTokenLifetimeMS
}
func (c *UserAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkURL(configErrs, "user_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "user_api.internal_api.connect", string(c.InternalAPI.Connect))
checkPositive(configErrs, "user_api.openid_token_lifetime_ms", c.OpenIDTokenLifetimeMS)
if c.Matrix.DatabaseOptions.ConnectionString == "" {
checkNotEmpty(configErrs, "user_api.account_database.connection_string", string(c.AccountDatabase.ConnectionString))
}
checkPositive(configErrs, "user_api.openid_token_lifetime_ms", c.OpenIDTokenLifetimeMS)
if isMonolith { // polylith required configs below
return
}
checkURL(configErrs, "user_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "user_api.internal_api.connect", string(c.InternalAPI.Connect))
}

25
syncapi/consumers/presence.go
View file

@ -138,9 +138,12 @@ func (s *PresenceConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
presence := msg.Header.Get("presence")
timestamp := msg.Header.Get("last_active_ts")
fromSync, _ := strconv.ParseBool(msg.Header.Get("from_sync"))
logrus.Debugf("syncAPI received presence event: %+v", msg.Header)
if fromSync { // do not process local presence changes; we already did this synchronously.
return true
}
ts, err := strconv.Atoi(timestamp)
if err != nil {
return true
@ -151,15 +154,19 @@ func (s *PresenceConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
newMsg := msg.Header.Get("status_msg")
statusMsg = &newMsg
}
// OK is already checked, so no need to do it again
// already checked, so no need to check error
p, _ := types.PresenceFromString(presence)
pos, err := s.db.UpdatePresence(ctx, userID, p, statusMsg, gomatrixserverlib.Timestamp(ts), fromSync)
if err != nil {
return true
}
s.stream.Advance(pos)
s.notifier.OnNewPresence(types.StreamingToken{PresencePosition: pos}, userID)
s.EmitPresence(ctx, userID, p, statusMsg, ts, fromSync)
return true
}
func (s *PresenceConsumer) EmitPresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, ts int, fromSync bool) {
pos, err := s.db.UpdatePresence(ctx, userID, presence, statusMsg, gomatrixserverlib.Timestamp(ts), fromSync)
if err != nil {
logrus.WithError(err).WithField("user", userID).WithField("presence", presence).Warn("failed to updated presence for user")
return
}
s.stream.Advance(pos)
s.notifier.OnNewPresence(types.StreamingToken{PresencePosition: pos}, userID)
}

6
syncapi/storage/storage_test.go
View file

@ -47,7 +47,7 @@ func MustWriteEvents(t *testing.T, db storage.Database, events []*gomatrixserver
func TestWriteEvents(t *testing.T) {
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
alice := test.NewUser()
alice := test.NewUser(t)
r := test.NewRoom(t, alice)
db, close := MustCreateDatabase(t, dbType)
defer close()
@ -60,7 +60,7 @@ func TestRecentEventsPDU(t *testing.T) {
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
db, close := MustCreateDatabase(t, dbType)
defer close()
alice := test.NewUser()
alice := test.NewUser(t)
// dummy room to make sure SQL queries are filtering on room ID
MustWriteEvents(t, db, test.NewRoom(t, alice).Events())
@ -163,7 +163,7 @@ func TestGetEventsInRangeWithTopologyToken(t *testing.T) {
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
db, close := MustCreateDatabase(t, dbType)
defer close()
alice := test.NewUser()
alice := test.NewUser(t)
r := test.NewRoom(t, alice)
for i := 0; i < 10; i++ {
r.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": fmt.Sprintf("hi %d", i)})

2
syncapi/storage/tables/output_room_events_test.go
View file

@ -45,7 +45,7 @@ func newOutputRoomEventsTable(t *testing.T, dbType test.DBType) (tables.Events,
func TestOutputRoomEventsTable(t *testing.T) {
ctx := context.Background()
alice := test.NewUser()
alice := test.NewUser(t)
room := test.NewRoom(t, alice)
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
tab, db, close := newOutputRoomEventsTable(t, dbType)

2
syncapi/storage/tables/topology_test.go
View file

@ -40,7 +40,7 @@ func newTopologyTable(t *testing.T, dbType test.DBType) (tables.Topology, *sql.D
func TestTopologyTable(t *testing.T) {
ctx := context.Background()
alice := test.NewUser()
alice := test.NewUser(t)
room := test.NewRoom(t, alice)
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
tab, db, close := newTopologyTable(t, dbType)

260
syncapi/sync/requestpool.go
View file

@ -53,19 +53,24 @@ type RequestPool struct {
streams *streams.Streams
Notifier *notifier.Notifier
producer PresencePublisher
consumer PresenceConsumer
}
type PresencePublisher interface {
SendPresence(userID string, presence types.Presence, statusMsg *string) error
}
type PresenceConsumer interface {
EmitPresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, ts int, fromSync bool)
}
// NewRequestPool makes a new RequestPool
func NewRequestPool(
db storage.Database, cfg *config.SyncAPI,
userAPI userapi.SyncUserAPI, keyAPI keyapi.SyncKeyAPI,
rsAPI roomserverAPI.SyncRoomserverAPI,
streams *streams.Streams, notifier *notifier.Notifier,
producer PresencePublisher, enableMetrics bool,
producer PresencePublisher, consumer PresenceConsumer, enableMetrics bool,
) *RequestPool {
if enableMetrics {
prometheus.MustRegister(
@ -83,6 +88,7 @@ func NewRequestPool(
streams: streams,
Notifier: notifier,
producer: producer,
consumer: consumer,
}
go rp.cleanLastSeen()
go rp.cleanPresence(db, time.Minute*5)
@ -160,6 +166,13 @@ func (rp *RequestPool) updatePresence(db storage.Presence, presence string, user
logrus.WithError(err).Error("Unable to publish presence message from sync")
return
}
// now synchronously update our view of the world. It's critical we do this before calculating
// the /sync response else we may not return presence: online immediately.
rp.consumer.EmitPresence(
context.Background(), userID, presenceID, newPresence.ClientFields.StatusMsg,
int(gomatrixserverlib.AsTimestamp(time.Now())), true,
)
}
func (rp *RequestPool) updateLastSeen(req *http.Request, device *userapi.Device) {
@ -238,122 +251,151 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
waitingSyncRequests.Inc()
defer waitingSyncRequests.Dec()
currentPos := rp.Notifier.CurrentPosition()
// loop until we get some data
for {
startTime := time.Now()
currentPos := rp.Notifier.CurrentPosition()
if !rp.shouldReturnImmediately(syncReq, currentPos) {
timer := time.NewTimer(syncReq.Timeout) // case of timeout=0 is handled above
defer timer.Stop()
// if the since token matches the current positions, wait via the notifier
if !rp.shouldReturnImmediately(syncReq, currentPos) {
timer := time.NewTimer(syncReq.Timeout) // case of timeout=0 is handled above
defer timer.Stop()
userStreamListener := rp.Notifier.GetListener(*syncReq)
defer userStreamListener.Close()
userStreamListener := rp.Notifier.GetListener(*syncReq)
defer userStreamListener.Close()
giveup := func() util.JSONResponse {
syncReq.Log.Debugln("Responding to sync since client gave up or timeout was reached")
syncReq.Response.NextBatch = syncReq.Since
// We should always try to include OTKs in sync responses, otherwise clients might upload keys
// even if that's not required. See also:
// https://github.com/matrix-org/synapse/blob/29f06704b8871a44926f7c99e73cf4a978fb8e81/synapse/rest/client/sync.py#L276-L281
err = internal.DeviceOTKCounts(syncReq.Context, rp.keyAPI, syncReq.Device.UserID, syncReq.Device.ID, syncReq.Response)
if err != nil {
syncReq.Log.WithError(err).Error("failed to get OTK counts")
giveup := func() util.JSONResponse {
syncReq.Log.Debugln("Responding to sync since client gave up or timeout was reached")
syncReq.Response.NextBatch = syncReq.Since
// We should always try to include OTKs in sync responses, otherwise clients might upload keys
// even if that's not required. See also:
// https://github.com/matrix-org/synapse/blob/29f06704b8871a44926f7c99e73cf4a978fb8e81/synapse/rest/client/sync.py#L276-L281
// Only try to get OTKs if the context isn't already done.
if syncReq.Context.Err() == nil {
err = internal.DeviceOTKCounts(syncReq.Context, rp.keyAPI, syncReq.Device.UserID, syncReq.Device.ID, syncReq.Response)
if err != nil && err != context.Canceled {
syncReq.Log.WithError(err).Warn("failed to get OTK counts")
}
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: syncReq.Response,
}
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: syncReq.Response,
select {
case <-syncReq.Context.Done(): // Caller gave up
return giveup()
case <-timer.C: // Timeout reached
return giveup()
case <-userStreamListener.GetNotifyChannel(syncReq.Since):
syncReq.Log.Debugln("Responding to sync after wake-up")
currentPos.ApplyUpdates(userStreamListener.GetSyncPosition())
}
} else {
syncReq.Log.WithField("currentPos", currentPos).Debugln("Responding to sync immediately")
}
if syncReq.Since.IsEmpty() {
// Complete sync
syncReq.Response.NextBatch = types.StreamingToken{
PDUPosition: rp.streams.PDUStreamProvider.CompleteSync(
syncReq.Context, syncReq,
),
TypingPosition: rp.streams.TypingStreamProvider.CompleteSync(
syncReq.Context, syncReq,
),
ReceiptPosition: rp.streams.ReceiptStreamProvider.CompleteSync(
syncReq.Context, syncReq,
),
InvitePosition: rp.streams.InviteStreamProvider.CompleteSync(
syncReq.Context, syncReq,
),
SendToDevicePosition: rp.streams.SendToDeviceStreamProvider.CompleteSync(
syncReq.Context, syncReq,
),
AccountDataPosition: rp.streams.AccountDataStreamProvider.CompleteSync(
syncReq.Context, syncReq,
),
NotificationDataPosition: rp.streams.NotificationDataStreamProvider.CompleteSync(
syncReq.Context, syncReq,
),
DeviceListPosition: rp.streams.DeviceListStreamProvider.CompleteSync(
syncReq.Context, syncReq,
),
PresencePosition: rp.streams.PresenceStreamProvider.CompleteSync(
syncReq.Context, syncReq,
),
}
} else {
// Incremental sync
syncReq.Response.NextBatch = types.StreamingToken{
PDUPosition: rp.streams.PDUStreamProvider.IncrementalSync(
syncReq.Context, syncReq,
syncReq.Since.PDUPosition, currentPos.PDUPosition,
),
TypingPosition: rp.streams.TypingStreamProvider.IncrementalSync(
syncReq.Context, syncReq,
syncReq.Since.TypingPosition, currentPos.TypingPosition,
),
ReceiptPosition: rp.streams.ReceiptStreamProvider.IncrementalSync(
syncReq.Context, syncReq,
syncReq.Since.ReceiptPosition, currentPos.ReceiptPosition,
),
InvitePosition: rp.streams.InviteStreamProvider.IncrementalSync(
syncReq.Context, syncReq,
syncReq.Since.InvitePosition, currentPos.InvitePosition,
),
SendToDevicePosition: rp.streams.SendToDeviceStreamProvider.IncrementalSync(
syncReq.Context, syncReq,
syncReq.Since.SendToDevicePosition, currentPos.SendToDevicePosition,
),
AccountDataPosition: rp.streams.AccountDataStreamProvider.IncrementalSync(
syncReq.Context, syncReq,
syncReq.Since.AccountDataPosition, currentPos.AccountDataPosition,
),
NotificationDataPosition: rp.streams.NotificationDataStreamProvider.IncrementalSync(
syncReq.Context, syncReq,
syncReq.Since.NotificationDataPosition, currentPos.NotificationDataPosition,
),
DeviceListPosition: rp.streams.DeviceListStreamProvider.IncrementalSync(
syncReq.Context, syncReq,
syncReq.Since.DeviceListPosition, currentPos.DeviceListPosition,
),
PresencePosition: rp.streams.PresenceStreamProvider.IncrementalSync(
syncReq.Context, syncReq,
syncReq.Since.PresencePosition, currentPos.PresencePosition,
),
}
// it's possible for there to be no updates for this user even though since < current pos,
// e.g busy servers with a quiet user. In this scenario, we don't want to return a no-op
// response immediately, so let's try this again but pretend they bumped their since token.
// If the incremental sync was processed very quickly then we expect the next loop to block
// with a notifier, but if things are slow it's entirely possible that currentPos is no
// longer the current position so we will hit this code path again. We need to do this and
// not return a no-op response because:
// - It's an inefficient use of bandwidth.
// - Some sytests which test 'waking up' sync rely on some sync requests to block, which
// they weren't always doing, resulting in flakey tests.
if !syncReq.Response.HasUpdates() {
syncReq.Since = currentPos
// do not loop again if the ?timeout= is 0 as that means "return immediately"
if syncReq.Timeout > 0 {
syncReq.Timeout = syncReq.Timeout - time.Since(startTime)
if syncReq.Timeout < 0 {
syncReq.Timeout = 0
}
continue
}
}
}
select {
case <-syncReq.Context.Done(): // Caller gave up
return giveup()
case <-timer.C: // Timeout reached
return giveup()
case <-userStreamListener.GetNotifyChannel(syncReq.Since):
syncReq.Log.Debugln("Responding to sync after wake-up")
currentPos.ApplyUpdates(userStreamListener.GetSyncPosition())
return util.JSONResponse{
Code: http.StatusOK,
JSON: syncReq.Response,
}
} else {
syncReq.Log.WithField("currentPos", currentPos).Debugln("Responding to sync immediately")
}
if syncReq.Since.IsEmpty() {
// Complete sync
syncReq.Response.NextBatch = types.StreamingToken{
PDUPosition: rp.streams.PDUStreamProvider.CompleteSync(
syncReq.Context, syncReq,
),
TypingPosition: rp.streams.TypingStreamProvider.CompleteSync(
syncReq.Context, syncReq,
),
ReceiptPosition: rp.streams.ReceiptStreamProvider.CompleteSync(
syncReq.Context, syncReq,
),
InvitePosition: rp.streams.InviteStreamProvider.CompleteSync(
syncReq.Context, syncReq,
),
SendToDevicePosition: rp.streams.SendToDeviceStreamProvider.CompleteSync(
syncReq.Context, syncReq,
),
AccountDataPosition: rp.streams.AccountDataStreamProvider.CompleteSync(
syncReq.Context, syncReq,
),
NotificationDataPosition: rp.streams.NotificationDataStreamProvider.CompleteSync(
syncReq.Context, syncReq,
),
DeviceListPosition: rp.streams.DeviceListStreamProvider.CompleteSync(
syncReq.Context, syncReq,
),
PresencePosition: rp.streams.PresenceStreamProvider.CompleteSync(
syncReq.Context, syncReq,
),
}
} else {
// Incremental sync
syncReq.Response.NextBatch = types.StreamingToken{
PDUPosition: rp.streams.PDUStreamProvider.IncrementalSync(
syncReq.Context, syncReq,
syncReq.Since.PDUPosition, currentPos.PDUPosition,
),
TypingPosition: rp.streams.TypingStreamProvider.IncrementalSync(
syncReq.Context, syncReq,
syncReq.Since.TypingPosition, currentPos.TypingPosition,
),
ReceiptPosition: rp.streams.ReceiptStreamProvider.IncrementalSync(
syncReq.Context, syncReq,
syncReq.Since.ReceiptPosition, currentPos.ReceiptPosition,
),
InvitePosition: rp.streams.InviteStreamProvider.IncrementalSync(
syncReq.Context, syncReq,
syncReq.Since.InvitePosition, currentPos.InvitePosition,
),
SendToDevicePosition: rp.streams.SendToDeviceStreamProvider.IncrementalSync(
syncReq.Context, syncReq,
syncReq.Since.SendToDevicePosition, currentPos.SendToDevicePosition,
),
AccountDataPosition: rp.streams.AccountDataStreamProvider.IncrementalSync(
syncReq.Context, syncReq,
syncReq.Since.AccountDataPosition, currentPos.AccountDataPosition,
),
NotificationDataPosition: rp.streams.NotificationDataStreamProvider.IncrementalSync(
syncReq.Context, syncReq,
syncReq.Since.NotificationDataPosition, currentPos.NotificationDataPosition,
),
DeviceListPosition: rp.streams.DeviceListStreamProvider.IncrementalSync(
syncReq.Context, syncReq,
syncReq.Since.DeviceListPosition, currentPos.DeviceListPosition,
),
PresencePosition: rp.streams.PresenceStreamProvider.IncrementalSync(
syncReq.Context, syncReq,
syncReq.Since.PresencePosition, currentPos.PresencePosition,
),
}
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: syncReq.Response,
}
}

8
syncapi/sync/requestpool_test.go
View file

@ -38,6 +38,12 @@ func (d dummyDB) MaxStreamPositionForPresence(ctx context.Context) (types.Stream
return 0, nil
}
type dummyConsumer struct{}
func (d dummyConsumer) EmitPresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, ts int, fromSync bool) {
}
func TestRequestPool_updatePresence(t *testing.T) {
type args struct {
presence string
@ -45,6 +51,7 @@ func TestRequestPool_updatePresence(t *testing.T) {
sleep time.Duration
}
publisher := &dummyPublisher{}
consumer := &dummyConsumer{}
syncMap := sync.Map{}
tests := []struct {
@ -101,6 +108,7 @@ func TestRequestPool_updatePresence(t *testing.T) {
rp := &RequestPool{
presence: &syncMap,
producer: publisher,
consumer: consumer,
cfg: &config.SyncAPI{
Matrix: &config.Global{
JetStream: config.JetStream{

20
syncapi/syncapi.go
View file

@ -64,8 +64,17 @@ func AddPublicRoutes(
Topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent),
JetStream: js,
}
presenceConsumer := consumers.NewPresenceConsumer(
base.ProcessContext, cfg, js, natsClient, syncDB,
notifier, streams.PresenceStreamProvider,
userAPI,
)
requestPool := sync.NewRequestPool(syncDB, cfg, userAPI, keyAPI, rsAPI, streams, notifier, federationPresenceProducer, base.EnableMetrics)
requestPool := sync.NewRequestPool(syncDB, cfg, userAPI, keyAPI, rsAPI, streams, notifier, federationPresenceProducer, presenceConsumer, base.EnableMetrics)
if err = presenceConsumer.Start(); err != nil {
logrus.WithError(err).Panicf("failed to start presence consumer")
}
userAPIStreamEventProducer := &producers.UserAPIStreamEventProducer{
JetStream: js,
@ -131,15 +140,6 @@ func AddPublicRoutes(
logrus.WithError(err).Panicf("failed to start receipts consumer")
}
presenceConsumer := consumers.NewPresenceConsumer(
base.ProcessContext, cfg, js, natsClient, syncDB,
notifier, streams.PresenceStreamProvider,
userAPI,
)
if err = presenceConsumer.Start(); err != nil {
logrus.WithError(err).Panicf("failed to start presence consumer")
}
routing.Setup(
base.PublicClientAPIMux, requestPool, syncDB, userAPI,
rsAPI, cfg, base.Caches,

70
syncapi/syncapi_test.go
View file

@ -15,9 +15,11 @@ import (
"github.com/matrix-org/dendrite/setup/jetstream"
"github.com/matrix-org/dendrite/syncapi/types"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/test/testrig"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/nats-io/nats.go"
"github.com/tidwall/gjson"
)
type syncRoomserverAPI struct {
@ -86,7 +88,7 @@ func TestSyncAPIAccessTokens(t *testing.T) {
}
func testSyncAccessTokens(t *testing.T, dbType test.DBType) {
user := test.NewUser()
user := test.NewUser(t)
room := test.NewRoom(t, user)
alice := userapi.Device{
ID: "ALICEID",
@ -96,14 +98,14 @@ func testSyncAccessTokens(t *testing.T, dbType test.DBType) {
AccountType: userapi.AccountTypeUser,
}
base, close := test.CreateBaseDendrite(t, dbType)
base, close := testrig.CreateBaseDendrite(t, dbType)
defer close()
jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream)
msgs := toNATSMsgs(t, base, room.Events())
AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, &syncKeyAPI{})
test.MustPublishMsgs(t, jsctx, msgs...)
testrig.MustPublishMsgs(t, jsctx, msgs...)
testCases := []struct {
name string
@ -173,7 +175,7 @@ func TestSyncAPICreateRoomSyncEarly(t *testing.T) {
}
func testSyncAPICreateRoomSyncEarly(t *testing.T, dbType test.DBType) {
user := test.NewUser()
user := test.NewUser(t)
room := test.NewRoom(t, user)
alice := userapi.Device{
ID: "ALICEID",
@ -183,7 +185,7 @@ func testSyncAPICreateRoomSyncEarly(t *testing.T, dbType test.DBType) {
AccountType: userapi.AccountTypeUser,
}
base, close := test.CreateBaseDendrite(t, dbType)
base, close := testrig.CreateBaseDendrite(t, dbType)
defer close()
jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
@ -198,7 +200,7 @@ func testSyncAPICreateRoomSyncEarly(t *testing.T, dbType test.DBType) {
sinceTokens := make([]string, len(msgs))
AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, &syncKeyAPI{})
for i, msg := range msgs {
test.MustPublishMsgs(t, jsctx, msg)
testrig.MustPublishMsgs(t, jsctx, msg)
time.Sleep(100 * time.Millisecond)
w := httptest.NewRecorder()
base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{
@ -255,6 +257,60 @@ func testSyncAPICreateRoomSyncEarly(t *testing.T, dbType test.DBType) {
}
}
// Test that if we hit /sync we get back presence: online, regardless of whether messages get delivered
// via NATS. Regression test for a flakey test "User sees their own presence in a sync"
func TestSyncAPIUpdatePresenceImmediately(t *testing.T) {
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
testSyncAPIUpdatePresenceImmediately(t, dbType)
})
}
func testSyncAPIUpdatePresenceImmediately(t *testing.T, dbType test.DBType) {
user := test.NewUser(t)
alice := userapi.Device{
ID: "ALICEID",
UserID: user.ID,
AccessToken: "ALICE_BEARER_TOKEN",
DisplayName: "Alice",
AccountType: userapi.AccountTypeUser,
}
base, close := testrig.CreateBaseDendrite(t, dbType)
base.Cfg.Global.Presence.EnableOutbound = true
base.Cfg.Global.Presence.EnableInbound = true
defer close()
jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream)
AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{}, &syncKeyAPI{})
w := httptest.NewRecorder()
base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{
"access_token": alice.AccessToken,
"timeout": "0",
"set_presence": "online",
})))
if w.Code != 200 {
t.Fatalf("got HTTP %d want %d", w.Code, 200)
}
var res types.Response
if err := json.NewDecoder(w.Body).Decode(&res); err != nil {
t.Errorf("failed to decode response body: %s", err)
}
if len(res.Presence.Events) != 1 {
t.Fatalf("expected 1 presence events, got: %+v", res.Presence.Events)
}
if res.Presence.Events[0].Sender != alice.UserID {
t.Errorf("sender: got %v want %v", res.Presence.Events[0].Sender, alice.UserID)
}
if res.Presence.Events[0].Type != "m.presence" {
t.Errorf("type: got %v want %v", res.Presence.Events[0].Type, "m.presence")
}
if gjson.ParseBytes(res.Presence.Events[0].Content).Get("presence").Str != "online" {
t.Errorf("content: not online, got %v", res.Presence.Events[0].Content)
}
}
func toNATSMsgs(t *testing.T, base *base.BaseDendrite, input []*gomatrixserverlib.HeaderedEvent) []*nats.Msg {
result := make([]*nats.Msg, len(input))
for i, ev := range input {
@ -262,7 +318,7 @@ func toNATSMsgs(t *testing.T, base *base.BaseDendrite, input []*gomatrixserverli
if ev.StateKey() != nil {
addsStateIDs = append(addsStateIDs, ev.EventID())
}
result[i] = test.NewOutputEventMsg(t, base, ev.RoomID(), api.OutputEvent{
result[i] = testrig.NewOutputEventMsg(t, base, ev.RoomID(), api.OutputEvent{
Type: rsapi.OutputTypeNewRoomEvent,
NewRoomEvent: &rsapi.OutputNewRoomEvent{
Event: ev,

13
syncapi/types/types.go
View file

@ -350,6 +350,19 @@ type Response struct {
DeviceListsOTKCount map[string]int `json:"device_one_time_keys_count,omitempty"`
}
func (r *Response) HasUpdates() bool {
// purposefully exclude DeviceListsOTKCount as we always include them
return (len(r.AccountData.Events) > 0 ||
len(r.Presence.Events) > 0 ||
len(r.Rooms.Invite) > 0 ||
len(r.Rooms.Join) > 0 ||
len(r.Rooms.Leave) > 0 ||
len(r.Rooms.Peek) > 0 ||
len(r.ToDevice.Events) > 0 ||
len(r.DeviceLists.Changed) > 0 ||
len(r.DeviceLists.Left) > 0)
}
// NewResponse creates an empty response with initialised maps.
func NewResponse() *Response {
res := Response{}

8
test/db.go
View file

@ -44,8 +44,9 @@ func fatalError(t *testing.T, format string, args ...interface{}) {
}
func createLocalDB(t *testing.T, dbName string) {
if !Quiet {
t.Log("Note: tests require a postgres install accessible to the current user")
if _, err := exec.LookPath("createdb"); err != nil {
fatalError(t, "Note: tests require a postgres install accessible to the current user")
return
}
createDB := exec.Command("createdb", dbName)
if !Quiet {
@ -63,6 +64,9 @@ func createRemoteDB(t *testing.T, dbName, user, connStr string) {
if err != nil {
fatalError(t, "failed to open postgres conn with connstr=%s : %s", connStr, err)
}
if err = db.Ping(); err != nil {
fatalError(t, "failed to open postgres conn with connstr=%s : %s", connStr, err)
}
_, err = db.Exec(fmt.Sprintf(`CREATE DATABASE %s;`, dbName))
if err != nil {
pqErr, ok := err.(*pq.Error)

18
test/event.go
View file

@ -52,6 +52,24 @@ func WithUnsigned(unsigned interface{}) eventModifier {
}
}
func WithKeyID(keyID gomatrixserverlib.KeyID) eventModifier {
return func(e *eventMods) {
e.keyID = keyID
}
}
func WithPrivateKey(pkey ed25519.PrivateKey) eventModifier {
return func(e *eventMods) {
e.privKey = pkey
}
}
func WithOrigin(origin gomatrixserverlib.ServerName) eventModifier {
return func(e *eventMods) {
e.origin = origin
}
}
// Reverse a list of events
func Reversed(in []*gomatrixserverlib.HeaderedEvent) []*gomatrixserverlib.HeaderedEvent {
out := make([]*gomatrixserverlib.HeaderedEvent, len(in))

47
test/http.go
View file

@ -2,10 +2,15 @@ package test
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"net/url"
"path/filepath"
"sync"
"testing"
)
@ -43,3 +48,45 @@ func NewRequest(t *testing.T, method, path string, opts ...HTTPRequestOpt) *http
}
return req
}
// ListenAndServe will listen on a random high-numbered port and attach the given router.
// Returns the base URL to send requests to. Call `cancel` to shutdown the server, which will block until it has closed.
func ListenAndServe(t *testing.T, router http.Handler, withTLS bool) (apiURL string, cancel func()) {
listener, err := net.Listen("tcp", ":0")
if err != nil {
t.Fatalf("failed to listen: %s", err)
}
port := listener.Addr().(*net.TCPAddr).Port
srv := http.Server{}
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
srv.Handler = router
var err error
if withTLS {
certFile := filepath.Join(t.TempDir(), "dendrite.cert")
keyFile := filepath.Join(t.TempDir(), "dendrite.key")
err = NewTLSKey(keyFile, certFile)
if err != nil {
t.Errorf("failed to make TLS key: %s", err)
return
}
err = srv.ServeTLS(listener, certFile, keyFile)
} else {
err = srv.Serve(listener)
}
if err != nil && err != http.ErrServerClosed {
t.Logf("Listen failed: %s", err)
}
}()
s := ""
if withTLS {
s = "s"
}
return fmt.Sprintf("http%s://localhost:%d", s, port), func() {
_ = srv.Shutdown(context.Background())
wg.Wait()
}
}

0
internal/test/keyring.go → test/keyring.go
View file

Some files were not shown because too many files have changed in this diff Show more