From 9bcd0a2105e127e12ca1bb22b6c5fe8c1923d07e Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Fri, 3 Mar 2023 14:03:17 +0100 Subject: [PATCH 1/3] Make redaction check easier to read (#2995) We need to check the redaction PL in Dendrite, if we do it in GMSL, we end up not sending the event to the output stream because it will be rejected. --------- Co-authored-by: kegsay --- roomserver/internal/input/input_events.go | 40 ++-- .../internal/perform/perform_backfill.go | 5 +- roomserver/roomserver_test.go | 194 +++++++++++++++++- roomserver/state/state.go | 41 ++++ roomserver/storage/interface.go | 5 +- roomserver/storage/shared/storage.go | 28 ++- 6 files changed, 285 insertions(+), 28 deletions(-) diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index ede345a93..d709541be 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -26,14 +26,14 @@ import ( "github.com/tidwall/gjson" - "github.com/matrix-org/dendrite/roomserver/internal/helpers" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/opentracing/opentracing-go" "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" + "github.com/matrix-org/dendrite/roomserver/internal/helpers" + userAPI "github.com/matrix-org/dendrite/userapi/api" fedapi "github.com/matrix-org/dendrite/federationapi/api" @@ -275,10 +275,8 @@ func (r *Inputer) processRoomEvent( // Check if the event is allowed by its auth events. If it isn't then // we consider the event to be "rejected" — it will still be persisted. - redactAllowed := true if err = gomatrixserverlib.Allowed(event, &authEvents); err != nil { isRejected = true - redactAllowed = false rejectionErr = err logger.WithError(rejectionErr).Warnf("Event %s not allowed by auth events", event.EventID()) } @@ -358,22 +356,6 @@ func (r *Inputer) processRoomEvent( return fmt.Errorf("updater.StoreEvent: %w", err) } - // if storing this event results in it being redacted then do so. - var ( - redactedEventID string - redactionEvent *gomatrixserverlib.Event - redactedEvent *gomatrixserverlib.Event - ) - if !isRejected && !isCreateEvent { - redactionEvent, redactedEvent, err = r.DB.MaybeRedactEvent(ctx, roomInfo, eventNID, event, redactAllowed) - if err != nil { - return err - } - if redactedEvent != nil { - redactedEventID = redactedEvent.EventID() - } - } - // For outliers we can stop after we've stored the event itself as it // doesn't have any associated state to store and we don't need to // notify anyone about it. @@ -402,6 +384,24 @@ func (r *Inputer) processRoomEvent( } } + // if storing this event results in it being redacted then do so. + // we do this after calculating state for this event as we may need to get power levels + var ( + redactedEventID string + redactionEvent *gomatrixserverlib.Event + redactedEvent *gomatrixserverlib.Event + ) + if !isRejected && !isCreateEvent { + resolver := state.NewStateResolution(r.DB, roomInfo) + redactionEvent, redactedEvent, err = r.DB.MaybeRedactEvent(ctx, roomInfo, eventNID, event, &resolver) + if err != nil { + return err + } + if redactedEvent != nil { + redactedEventID = redactedEvent.EventID() + } + } + // We stop here if the event is rejected: We've stored it but won't update // forward extremities or notify downstream components about it. switch { diff --git a/roomserver/internal/perform/perform_backfill.go b/roomserver/internal/perform/perform_backfill.go index 411f4202b..07ad28b82 100644 --- a/roomserver/internal/perform/perform_backfill.go +++ b/roomserver/internal/perform/perform_backfill.go @@ -26,6 +26,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/auth" "github.com/matrix-org/dendrite/roomserver/internal/helpers" + "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -629,7 +630,9 @@ func persistEvents(ctx context.Context, db storage.Database, events []*gomatrixs continue } - _, redactedEvent, err := db.MaybeRedactEvent(ctx, roomInfo, eventNID, ev.Unwrap(), true) + resolver := state.NewStateResolution(db, roomInfo) + + _, redactedEvent, err := db.MaybeRedactEvent(ctx, roomInfo, eventNID, ev.Unwrap(), &resolver) if err != nil { logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to redact event") continue diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go index cfa27e541..a3ca5909e 100644 --- a/roomserver/roomserver_test.go +++ b/roomserver/roomserver_test.go @@ -2,19 +2,25 @@ package roomserver_test import ( "context" + "crypto/ed25519" "reflect" "testing" "time" + "github.com/stretchr/testify/assert" + + "github.com/matrix-org/dendrite/roomserver/state" + "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/userapi" userAPI "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/syncapi" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/roomserver/api" @@ -379,3 +385,189 @@ func TestPurgeRoom(t *testing.T) { } }) } + +type fledglingEvent struct { + Type string + StateKey *string + Sender string + RoomID string + Redacts string + Depth int64 + PrevEvents []interface{} +} + +func mustCreateEvent(t *testing.T, ev fledglingEvent) (result *gomatrixserverlib.HeaderedEvent) { + t.Helper() + roomVer := gomatrixserverlib.RoomVersionV9 + seed := make([]byte, ed25519.SeedSize) // zero seed + key := ed25519.NewKeyFromSeed(seed) + eb := gomatrixserverlib.EventBuilder{ + Sender: ev.Sender, + Type: ev.Type, + StateKey: ev.StateKey, + RoomID: ev.RoomID, + Redacts: ev.Redacts, + Depth: ev.Depth, + PrevEvents: ev.PrevEvents, + } + err := eb.SetContent(map[string]interface{}{}) + if err != nil { + t.Fatalf("mustCreateEvent: failed to marshal event content %v", err) + } + signedEvent, err := eb.Build(time.Now(), "localhost", "ed25519:test", key, roomVer) + if err != nil { + t.Fatalf("mustCreateEvent: failed to sign event: %s", err) + } + h := signedEvent.Headered(roomVer) + return h +} + +func TestRedaction(t *testing.T) { + alice := test.NewUser(t) + bob := test.NewUser(t) + charlie := test.NewUser(t, test.WithSigningServer("notlocalhost", "abc", test.PrivateKeyB)) + + testCases := []struct { + name string + additionalEvents func(t *testing.T, room *test.Room) + wantRedacted bool + }{ + { + name: "can redact own message", + wantRedacted: true, + additionalEvents: func(t *testing.T, room *test.Room) { + redactedEvent := room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "hello world"}) + + builderEv := mustCreateEvent(t, fledglingEvent{ + Type: gomatrixserverlib.MRoomRedaction, + Sender: alice.ID, + RoomID: room.ID, + Redacts: redactedEvent.EventID(), + Depth: redactedEvent.Depth() + 1, + PrevEvents: []interface{}{redactedEvent.EventID()}, + }) + room.InsertEvent(t, builderEv.Headered(gomatrixserverlib.RoomVersionV9)) + }, + }, + { + name: "can redact others message, allowed by PL", + wantRedacted: true, + additionalEvents: func(t *testing.T, room *test.Room) { + redactedEvent := room.CreateAndInsert(t, bob, "m.room.message", map[string]interface{}{"body": "hello world"}) + + builderEv := mustCreateEvent(t, fledglingEvent{ + Type: gomatrixserverlib.MRoomRedaction, + Sender: alice.ID, + RoomID: room.ID, + Redacts: redactedEvent.EventID(), + Depth: redactedEvent.Depth() + 1, + PrevEvents: []interface{}{redactedEvent.EventID()}, + }) + room.InsertEvent(t, builderEv.Headered(gomatrixserverlib.RoomVersionV9)) + }, + }, + { + name: "can redact others message, same server", + wantRedacted: true, + additionalEvents: func(t *testing.T, room *test.Room) { + redactedEvent := room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "hello world"}) + + builderEv := mustCreateEvent(t, fledglingEvent{ + Type: gomatrixserverlib.MRoomRedaction, + Sender: bob.ID, + RoomID: room.ID, + Redacts: redactedEvent.EventID(), + Depth: redactedEvent.Depth() + 1, + PrevEvents: []interface{}{redactedEvent.EventID()}, + }) + room.InsertEvent(t, builderEv.Headered(gomatrixserverlib.RoomVersionV9)) + }, + }, + { + name: "can not redact others message, missing PL", + additionalEvents: func(t *testing.T, room *test.Room) { + redactedEvent := room.CreateAndInsert(t, bob, "m.room.message", map[string]interface{}{"body": "hello world"}) + + builderEv := mustCreateEvent(t, fledglingEvent{ + Type: gomatrixserverlib.MRoomRedaction, + Sender: charlie.ID, + RoomID: room.ID, + Redacts: redactedEvent.EventID(), + Depth: redactedEvent.Depth() + 1, + PrevEvents: []interface{}{redactedEvent.EventID()}, + }) + room.InsertEvent(t, builderEv.Headered(gomatrixserverlib.RoomVersionV9)) + }, + }, + } + + ctx := context.Background() + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + _, db, close := mustCreateDatabase(t, dbType) + defer close() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + authEvents := []types.EventNID{} + var roomInfo *types.RoomInfo + var err error + + room := test.NewRoom(t, alice, test.RoomPreset(test.PresetPublicChat)) + room.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{ + "membership": "join", + }, test.WithStateKey(bob.ID)) + room.CreateAndInsert(t, charlie, gomatrixserverlib.MRoomMember, map[string]interface{}{ + "membership": "join", + }, test.WithStateKey(charlie.ID)) + + if tc.additionalEvents != nil { + tc.additionalEvents(t, room) + } + + for _, ev := range room.Events() { + roomInfo, err = db.GetOrCreateRoomInfo(ctx, ev.Event) + assert.NoError(t, err) + assert.NotNil(t, roomInfo) + evTypeNID, err := db.GetOrCreateEventTypeNID(ctx, ev.Type()) + assert.NoError(t, err) + + stateKeyNID, err := db.GetOrCreateEventStateKeyNID(ctx, ev.StateKey()) + assert.NoError(t, err) + + eventNID, stateAtEvent, err := db.StoreEvent(ctx, ev.Event, roomInfo, evTypeNID, stateKeyNID, authEvents, false) + assert.NoError(t, err) + if ev.StateKey() != nil { + authEvents = append(authEvents, eventNID) + } + + // Calculate the snapshotNID etc. + plResolver := state.NewStateResolution(db, roomInfo) + stateAtEvent.BeforeStateSnapshotNID, err = plResolver.CalculateAndStoreStateBeforeEvent(ctx, ev.Event, false) + assert.NoError(t, err) + + // Update the room + updater, err := db.GetRoomUpdater(ctx, roomInfo) + assert.NoError(t, err) + err = updater.SetState(ctx, eventNID, stateAtEvent.BeforeStateSnapshotNID) + assert.NoError(t, err) + err = updater.Commit() + assert.NoError(t, err) + + _, redactedEvent, err := db.MaybeRedactEvent(ctx, roomInfo, eventNID, ev.Event, &plResolver) + assert.NoError(t, err) + if redactedEvent != nil { + assert.Equal(t, ev.Redacts(), redactedEvent.EventID()) + } + if ev.Type() == gomatrixserverlib.MRoomRedaction { + nids, err := db.EventNIDs(ctx, []string{ev.Redacts()}) + assert.NoError(t, err) + evs, err := db.Events(ctx, roomInfo, []types.EventNID{nids[ev.Redacts()].EventNID}) + assert.NoError(t, err) + assert.Equal(t, 1, len(evs)) + assert.Equal(t, tc.wantRedacted, evs[0].Redacted()) + } + } + }) + } + }) +} diff --git a/roomserver/state/state.go b/roomserver/state/state.go index 9af2f8577..47e1488df 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -59,6 +59,47 @@ func NewStateResolution(db StateResolutionStorage, roomInfo *types.RoomInfo) Sta } } +type PowerLevelResolver interface { + Resolve(ctx context.Context, eventID string) (*gomatrixserverlib.PowerLevelContent, error) +} + +func (p *StateResolution) Resolve(ctx context.Context, eventID string) (*gomatrixserverlib.PowerLevelContent, error) { + stateEntries, err := p.LoadStateAtEvent(ctx, eventID) + if err != nil { + return nil, err + } + + wantTuple := types.StateKeyTuple{ + EventTypeNID: types.MRoomPowerLevelsNID, + EventStateKeyNID: types.EmptyStateKeyNID, + } + + var plNID types.EventNID + for _, entry := range stateEntries { + if entry.StateKeyTuple == wantTuple { + plNID = entry.EventNID + break + } + } + if plNID == 0 { + return nil, fmt.Errorf("unable to find power level event") + } + + events, err := p.db.Events(ctx, p.roomInfo, []types.EventNID{plNID}) + if err != nil { + return nil, err + } + if len(events) == 0 { + return nil, fmt.Errorf("unable to find power level event") + } + powerlevels, err := events[0].PowerLevels() + if err != nil { + return nil, err + } + + return powerlevels, nil +} + // LoadStateAtSnapshot loads the full state of a room at a particular snapshot. // This is typically the state before an event or the current state of a room. // Returns a sorted list of state entries or an error if there was a problem talking to the database. diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index a41a8a9b4..5b90f8b33 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -19,6 +19,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -184,7 +185,7 @@ type Database interface { GetOrCreateEventTypeNID(ctx context.Context, eventType string) (eventTypeNID types.EventTypeNID, err error) GetOrCreateEventStateKeyNID(ctx context.Context, eventStateKey *string) (types.EventStateKeyNID, error) MaybeRedactEvent( - ctx context.Context, roomInfo *types.RoomInfo, eventNID types.EventNID, event *gomatrixserverlib.Event, redactAllowed bool, + ctx context.Context, roomInfo *types.RoomInfo, eventNID types.EventNID, event *gomatrixserverlib.Event, plResolver state.PowerLevelResolver, ) (*gomatrixserverlib.Event, *gomatrixserverlib.Event, error) } @@ -226,7 +227,7 @@ type EventDatabase interface { // MaybeRedactEvent returns the redaction event and the redacted event if this call resulted in a redaction, else an error // (nil if there was nothing to do) MaybeRedactEvent( - ctx context.Context, roomInfo *types.RoomInfo, eventNID types.EventNID, event *gomatrixserverlib.Event, redactAllowed bool, + ctx context.Context, roomInfo *types.RoomInfo, eventNID types.EventNID, event *gomatrixserverlib.Event, plResolver state.PowerLevelResolver, ) (*gomatrixserverlib.Event, *gomatrixserverlib.Event, error) StoreEvent(ctx context.Context, event *gomatrixserverlib.Event, roomInfo *types.RoomInfo, eventTypeNID types.EventTypeNID, eventStateKeyNID types.EventStateKeyNID, authEventNIDs []types.EventNID, isRejected bool) (types.EventNID, types.StateAtEvent, error) } diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index be3f228d7..9ecdcafb3 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -13,6 +13,7 @@ import ( "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -660,6 +661,12 @@ func (d *Database) GetOrCreateRoomInfo(ctx context.Context, event *gomatrixserve if roomVersion, err = extractRoomVersionFromCreateEvent(event); err != nil { return nil, fmt.Errorf("extractRoomVersionFromCreateEvent: %w", err) } + if roomVersion == "" { + rv, ok := d.Cache.GetRoomVersion(event.RoomID()) + if ok { + roomVersion = rv + } + } var roomNID types.RoomNID err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { roomNID, err = d.assignRoomNID(ctx, txn, event.RoomID(), roomVersion) @@ -668,6 +675,9 @@ func (d *Database) GetOrCreateRoomInfo(ctx context.Context, event *gomatrixserve } return nil }) + if roomVersion != "" { + d.Cache.StoreRoomVersion(event.RoomID(), roomVersion) + } return &types.RoomInfo{ RoomVersion: roomVersion, RoomNID: roomNID, @@ -838,6 +848,7 @@ func (d *Database) assignRoomNID( return 0, err } d.Cache.StoreRoomServerRoomID(roomNID, roomID) + d.Cache.StoreRoomVersion(roomID, roomVersion) return roomNID, nil } @@ -926,7 +937,7 @@ func extractRoomVersionFromCreateEvent(event *gomatrixserverlib.Event) ( // // Returns the redaction event and the redacted event if this call resulted in a redaction. func (d *EventDatabase) MaybeRedactEvent( - ctx context.Context, roomInfo *types.RoomInfo, eventNID types.EventNID, event *gomatrixserverlib.Event, redactAllowed bool, + ctx context.Context, roomInfo *types.RoomInfo, eventNID types.EventNID, event *gomatrixserverlib.Event, plResolver state.PowerLevelResolver, ) (*gomatrixserverlib.Event, *gomatrixserverlib.Event, error) { var ( redactionEvent, redactedEvent *types.Event @@ -966,11 +977,20 @@ func (d *EventDatabase) MaybeRedactEvent( return nil } - // 1. The power level of the redaction event’s sender is greater than or equal to the redact level. (redactAllowed) - // 2. The domain of the redaction event’s sender matches that of the original event’s sender. _, sender1, _ := gomatrixserverlib.SplitID('@', redactedEvent.Sender()) _, sender2, _ := gomatrixserverlib.SplitID('@', redactionEvent.Sender()) - if !redactAllowed || sender1 != sender2 { + var powerlevels *gomatrixserverlib.PowerLevelContent + powerlevels, err = plResolver.Resolve(ctx, redactionEvent.EventID()) + if err != nil { + return err + } + + switch { + case powerlevels.UserLevel(redactionEvent.Sender()) >= powerlevels.Redact: + // 1. The power level of the redaction event’s sender is greater than or equal to the redact level. + case sender1 == sender2: + // 2. The domain of the redaction event’s sender matches that of the original event’s sender. + default: ignoreRedaction = true return nil } From 56b28b01dbfb50af70c8d764369baa8e6aa6b6e6 Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Fri, 3 Mar 2023 14:49:41 +0100 Subject: [PATCH 2/3] Update the cache with the redacted event --- roomserver/storage/shared/storage.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index 9ecdcafb3..106a8244e 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -1019,6 +1019,9 @@ func (d *EventDatabase) MaybeRedactEvent( if err != nil { return fmt.Errorf("d.RedactionsTable.MarkRedactionValidated: %w", err) } + + d.Cache.StoreRoomServerEvent(redactedEvent.EventNID, redactedEvent.Event) + return nil }) if wErr != nil { From 7fc839f7519fe9af10ce438df6270cba96148872 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Sun, 5 Mar 2023 12:42:38 -0500 Subject: [PATCH 3/3] Update admin-promotion instructions; clarify ID for evacuation (#2997) Table name has changed since instructions were written. There's probably a better way to describe how to get the internal room ID than I've attempted here, so feel free to adjust as needed. (It may even be good to show an example of what an internal room ID looks like, e.g. `!nc93825:example.com`) ### Pull Request Checklist * [x] I have added Go unit tests or [Complement integration tests](https://github.com/matrix-org/complement) for this PR _or_ I have justified why this PR doesn't need tests * Doc-only change * [x] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) Signed-off-by: `Tim McCormack ` --------- Co-authored-by: Till <2353100+S7evinK@users.noreply.github.com> --- docs/administration/4_adminapi.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/administration/4_adminapi.md b/docs/administration/4_adminapi.md index c521cbc90..46cfac220 100644 --- a/docs/administration/4_adminapi.md +++ b/docs/administration/4_adminapi.md @@ -22,12 +22,12 @@ curl --header "Authorization: Bearer " -X `Help & About` -> `Advanced` -> `Access Token`. Be aware that an `access_token` allows a client to perform actions as an user and should be kept **secret**. -The user must be an administrator in the `account_accounts` table in order to use these endpoints. +The user must be an administrator in the `userapi_accounts` table in order to use these endpoints. -Existing user accounts can be set to administrative accounts by changing `account_type` to `3` in `account_accounts` +Existing user accounts can be set to administrative accounts by changing `account_type` to `3` in `userapi_accounts` ``` -UPDATE account_accounts SET account_type = 3 WHERE localpart = '$localpart'; +UPDATE userapi_accounts SET account_type = 3 WHERE localpart = '$localpart'; ``` Where `$localpart` is the username only (e.g. `alice`). @@ -38,6 +38,9 @@ This endpoint will instruct Dendrite to part all local users from the given `roo in the URL. It may take some time to complete. A JSON body will be returned containing the user IDs of all affected users. +If the room has an alias set (e.g. is published), the room's ID will not be visible in the URL, but it can +be found as the room's "internal ID" in Element Web (Settings -> Advanced) + ## GET `/_dendrite/admin/evacuateUser/{userID}` This endpoint will instruct Dendrite to part the given local `userID` in the URL from