Merge branch 'main' of github.com:matrix-org/dendrite into gh-pages

docs/administration/4_adminapi.md

@@ -22,12 +22,12 @@ curl --header "Authorization: Bearer <access_token>" -X <POST|GET|PUT> <Endpoint
 An `access_token` can be obtained through most Element-based matrix clients by going to `Settings` -> `Help & About` -> `Advanced` -> `Access Token`.
 Be aware that an `access_token` allows a client to perform actions as an user and should be kept **secret**.
 
-The user must be an administrator in the `account_accounts` table in order to use these endpoints.
+The user must be an administrator in the `userapi_accounts` table in order to use these endpoints.
 
-Existing user accounts can be set to administrative accounts by changing `account_type` to `3` in `account_accounts`
+Existing user accounts can be set to administrative accounts by changing `account_type` to `3` in `userapi_accounts`
 
 ```
-UPDATE account_accounts SET account_type = 3 WHERE localpart = '$localpart';
+UPDATE userapi_accounts SET account_type = 3 WHERE localpart = '$localpart';
 ```
 
 Where `$localpart` is the username only (e.g. `alice`).
@@ -38,6 +38,9 @@ This endpoint will instruct Dendrite to part all local users from the given `roo
 in the URL. It may take some time to complete. A JSON body will be returned containing
 the user IDs of all affected users.
 
+If the room has an alias set (e.g. is published), the room's ID will not be visible in the URL, but it can
+be found as the room's "internal ID" in Element Web (Settings -> Advanced)
+
 ## GET `/_dendrite/admin/evacuateUser/{userID}`
 
 This endpoint will instruct Dendrite to part the given local `userID` in the URL from

roomserver/internal/input/input_events.go

@@ -26,14 +26,14 @@ import (
 
 	"github.com/tidwall/gjson"
 
-	"github.com/matrix-org/dendrite/roomserver/internal/helpers"
-
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/opentracing/opentracing-go"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 
+	"github.com/matrix-org/dendrite/roomserver/internal/helpers"
+
 	userAPI "github.com/matrix-org/dendrite/userapi/api"
 
 	fedapi "github.com/matrix-org/dendrite/federationapi/api"
@@ -275,10 +275,8 @@ func (r *Inputer) processRoomEvent(
 
 	// Check if the event is allowed by its auth events. If it isn't then
 	// we consider the event to be "rejected" — it will still be persisted.
-	redactAllowed := true
 	if err = gomatrixserverlib.Allowed(event, &authEvents); err != nil {
 		isRejected = true
-		redactAllowed = false
 		rejectionErr = err
 		logger.WithError(rejectionErr).Warnf("Event %s not allowed by auth events", event.EventID())
 	}
@@ -358,22 +356,6 @@ func (r *Inputer) processRoomEvent(
 		return fmt.Errorf("updater.StoreEvent: %w", err)
 	}
 
-	// if storing this event results in it being redacted then do so.
-	var (
-		redactedEventID string
-		redactionEvent  *gomatrixserverlib.Event
-		redactedEvent   *gomatrixserverlib.Event
-	)
-	if !isRejected && !isCreateEvent {
-		redactionEvent, redactedEvent, err = r.DB.MaybeRedactEvent(ctx, roomInfo, eventNID, event, redactAllowed)
-		if err != nil {
-			return err
-		}
-		if redactedEvent != nil {
-			redactedEventID = redactedEvent.EventID()
-		}
-	}
-
 	// For outliers we can stop after we've stored the event itself as it
 	// doesn't have any associated state to store and we don't need to
 	// notify anyone about it.
@@ -402,6 +384,24 @@ func (r *Inputer) processRoomEvent(
 		}
 	}
 
+	// if storing this event results in it being redacted then do so.
+	// we do this after calculating state for this event as we may need to get power levels
+	var (
+		redactedEventID string
+		redactionEvent  *gomatrixserverlib.Event
+		redactedEvent   *gomatrixserverlib.Event
+	)
+	if !isRejected && !isCreateEvent {
+		resolver := state.NewStateResolution(r.DB, roomInfo)
+		redactionEvent, redactedEvent, err = r.DB.MaybeRedactEvent(ctx, roomInfo, eventNID, event, &resolver)
+		if err != nil {
+			return err
+		}
+		if redactedEvent != nil {
+			redactedEventID = redactedEvent.EventID()
+		}
+	}
+
 	// We stop here if the event is rejected: We've stored it but won't update
 	// forward extremities or notify downstream components about it.
 	switch {

roomserver/internal/perform/perform_backfill.go

@@ -26,6 +26,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/auth"
 	"github.com/matrix-org/dendrite/roomserver/internal/helpers"
+	"github.com/matrix-org/dendrite/roomserver/state"
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/roomserver/types"
 )
@@ -629,7 +630,9 @@ func persistEvents(ctx context.Context, db storage.Database, events []*gomatrixs
 			continue
 		}
 
-		_, redactedEvent, err := db.MaybeRedactEvent(ctx, roomInfo, eventNID, ev.Unwrap(), true)
+		resolver := state.NewStateResolution(db, roomInfo)
+
+		_, redactedEvent, err := db.MaybeRedactEvent(ctx, roomInfo, eventNID, ev.Unwrap(), &resolver)
 		if err != nil {
 			logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to redact event")
 			continue

roomserver/roomserver_test.go

@@ -2,19 +2,25 @@ package roomserver_test
 
 import (
 	"context"
+	"crypto/ed25519"
 	"reflect"
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
+
+	"github.com/matrix-org/dendrite/roomserver/state"
+	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/userapi"
 
 	userAPI "github.com/matrix-org/dendrite/userapi/api"
 
+	"github.com/matrix-org/gomatrixserverlib"
+
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/syncapi"
-	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/roomserver/api"
@@ -379,3 +385,189 @@ func TestPurgeRoom(t *testing.T) {
 		}
 	})
 }
+
+type fledglingEvent struct {
+	Type       string
+	StateKey   *string
+	Sender     string
+	RoomID     string
+	Redacts    string
+	Depth      int64
+	PrevEvents []interface{}
+}
+
+func mustCreateEvent(t *testing.T, ev fledglingEvent) (result *gomatrixserverlib.HeaderedEvent) {
+	t.Helper()
+	roomVer := gomatrixserverlib.RoomVersionV9
+	seed := make([]byte, ed25519.SeedSize) // zero seed
+	key := ed25519.NewKeyFromSeed(seed)
+	eb := gomatrixserverlib.EventBuilder{
+		Sender:     ev.Sender,
+		Type:       ev.Type,
+		StateKey:   ev.StateKey,
+		RoomID:     ev.RoomID,
+		Redacts:    ev.Redacts,
+		Depth:      ev.Depth,
+		PrevEvents: ev.PrevEvents,
+	}
+	err := eb.SetContent(map[string]interface{}{})
+	if err != nil {
+		t.Fatalf("mustCreateEvent: failed to marshal event content %v", err)
+	}
+	signedEvent, err := eb.Build(time.Now(), "localhost", "ed25519:test", key, roomVer)
+	if err != nil {
+		t.Fatalf("mustCreateEvent: failed to sign event: %s", err)
+	}
+	h := signedEvent.Headered(roomVer)
+	return h
+}
+
+func TestRedaction(t *testing.T) {
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+	charlie := test.NewUser(t, test.WithSigningServer("notlocalhost", "abc", test.PrivateKeyB))
+
+	testCases := []struct {
+		name             string
+		additionalEvents func(t *testing.T, room *test.Room)
+		wantRedacted     bool
+	}{
+		{
+			name:         "can redact own message",
+			wantRedacted: true,
+			additionalEvents: func(t *testing.T, room *test.Room) {
+				redactedEvent := room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "hello world"})
+
+				builderEv := mustCreateEvent(t, fledglingEvent{
+					Type:       gomatrixserverlib.MRoomRedaction,
+					Sender:     alice.ID,
+					RoomID:     room.ID,
+					Redacts:    redactedEvent.EventID(),
+					Depth:      redactedEvent.Depth() + 1,
+					PrevEvents: []interface{}{redactedEvent.EventID()},
+				})
+				room.InsertEvent(t, builderEv.Headered(gomatrixserverlib.RoomVersionV9))
+			},
+		},
+		{
+			name:         "can redact others message, allowed by PL",
+			wantRedacted: true,
+			additionalEvents: func(t *testing.T, room *test.Room) {
+				redactedEvent := room.CreateAndInsert(t, bob, "m.room.message", map[string]interface{}{"body": "hello world"})
+
+				builderEv := mustCreateEvent(t, fledglingEvent{
+					Type:       gomatrixserverlib.MRoomRedaction,
+					Sender:     alice.ID,
+					RoomID:     room.ID,
+					Redacts:    redactedEvent.EventID(),
+					Depth:      redactedEvent.Depth() + 1,
+					PrevEvents: []interface{}{redactedEvent.EventID()},
+				})
+				room.InsertEvent(t, builderEv.Headered(gomatrixserverlib.RoomVersionV9))
+			},
+		},
+		{
+			name:         "can redact others message, same server",
+			wantRedacted: true,
+			additionalEvents: func(t *testing.T, room *test.Room) {
+				redactedEvent := room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "hello world"})
+
+				builderEv := mustCreateEvent(t, fledglingEvent{
+					Type:       gomatrixserverlib.MRoomRedaction,
+					Sender:     bob.ID,
+					RoomID:     room.ID,
+					Redacts:    redactedEvent.EventID(),
+					Depth:      redactedEvent.Depth() + 1,
+					PrevEvents: []interface{}{redactedEvent.EventID()},
+				})
+				room.InsertEvent(t, builderEv.Headered(gomatrixserverlib.RoomVersionV9))
+			},
+		},
+		{
+			name: "can not redact others message, missing PL",
+			additionalEvents: func(t *testing.T, room *test.Room) {
+				redactedEvent := room.CreateAndInsert(t, bob, "m.room.message", map[string]interface{}{"body": "hello world"})
+
+				builderEv := mustCreateEvent(t, fledglingEvent{
+					Type:       gomatrixserverlib.MRoomRedaction,
+					Sender:     charlie.ID,
+					RoomID:     room.ID,
+					Redacts:    redactedEvent.EventID(),
+					Depth:      redactedEvent.Depth() + 1,
+					PrevEvents: []interface{}{redactedEvent.EventID()},
+				})
+				room.InsertEvent(t, builderEv.Headered(gomatrixserverlib.RoomVersionV9))
+			},
+		},
+	}
+
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		_, db, close := mustCreateDatabase(t, dbType)
+		defer close()
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				authEvents := []types.EventNID{}
+				var roomInfo *types.RoomInfo
+				var err error
+
+				room := test.NewRoom(t, alice, test.RoomPreset(test.PresetPublicChat))
+				room.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{
+					"membership": "join",
+				}, test.WithStateKey(bob.ID))
+				room.CreateAndInsert(t, charlie, gomatrixserverlib.MRoomMember, map[string]interface{}{
+					"membership": "join",
+				}, test.WithStateKey(charlie.ID))
+
+				if tc.additionalEvents != nil {
+					tc.additionalEvents(t, room)
+				}
+
+				for _, ev := range room.Events() {
+					roomInfo, err = db.GetOrCreateRoomInfo(ctx, ev.Event)
+					assert.NoError(t, err)
+					assert.NotNil(t, roomInfo)
+					evTypeNID, err := db.GetOrCreateEventTypeNID(ctx, ev.Type())
+					assert.NoError(t, err)
+
+					stateKeyNID, err := db.GetOrCreateEventStateKeyNID(ctx, ev.StateKey())
+					assert.NoError(t, err)
+
+					eventNID, stateAtEvent, err := db.StoreEvent(ctx, ev.Event, roomInfo, evTypeNID, stateKeyNID, authEvents, false)
+					assert.NoError(t, err)
+					if ev.StateKey() != nil {
+						authEvents = append(authEvents, eventNID)
+					}
+
+					// Calculate the snapshotNID etc.
+					plResolver := state.NewStateResolution(db, roomInfo)
+					stateAtEvent.BeforeStateSnapshotNID, err = plResolver.CalculateAndStoreStateBeforeEvent(ctx, ev.Event, false)
+					assert.NoError(t, err)
+
+					// Update the room
+					updater, err := db.GetRoomUpdater(ctx, roomInfo)
+					assert.NoError(t, err)
+					err = updater.SetState(ctx, eventNID, stateAtEvent.BeforeStateSnapshotNID)
+					assert.NoError(t, err)
+					err = updater.Commit()
+					assert.NoError(t, err)
+
+					_, redactedEvent, err := db.MaybeRedactEvent(ctx, roomInfo, eventNID, ev.Event, &plResolver)
+					assert.NoError(t, err)
+					if redactedEvent != nil {
+						assert.Equal(t, ev.Redacts(), redactedEvent.EventID())
+					}
+					if ev.Type() == gomatrixserverlib.MRoomRedaction {
+						nids, err := db.EventNIDs(ctx, []string{ev.Redacts()})
+						assert.NoError(t, err)
+						evs, err := db.Events(ctx, roomInfo, []types.EventNID{nids[ev.Redacts()].EventNID})
+						assert.NoError(t, err)
+						assert.Equal(t, 1, len(evs))
+						assert.Equal(t, tc.wantRedacted, evs[0].Redacted())
+					}
+				}
+			})
+		}
+	})
+}

roomserver/state/state.go

@@ -59,6 +59,47 @@ func NewStateResolution(db StateResolutionStorage, roomInfo *types.RoomInfo) Sta
 	}
 }
 
+type PowerLevelResolver interface {
+	Resolve(ctx context.Context, eventID string) (*gomatrixserverlib.PowerLevelContent, error)
+}
+
+func (p *StateResolution) Resolve(ctx context.Context, eventID string) (*gomatrixserverlib.PowerLevelContent, error) {
+	stateEntries, err := p.LoadStateAtEvent(ctx, eventID)
+	if err != nil {
+		return nil, err
+	}
+
+	wantTuple := types.StateKeyTuple{
+		EventTypeNID:     types.MRoomPowerLevelsNID,
+		EventStateKeyNID: types.EmptyStateKeyNID,
+	}
+
+	var plNID types.EventNID
+	for _, entry := range stateEntries {
+		if entry.StateKeyTuple == wantTuple {
+			plNID = entry.EventNID
+			break
+		}
+	}
+	if plNID == 0 {
+		return nil, fmt.Errorf("unable to find power level event")
+	}
+
+	events, err := p.db.Events(ctx, p.roomInfo, []types.EventNID{plNID})
+	if err != nil {
+		return nil, err
+	}
+	if len(events) == 0 {
+		return nil, fmt.Errorf("unable to find power level event")
+	}
+	powerlevels, err := events[0].PowerLevels()
+	if err != nil {
+		return nil, err
+	}
+
+	return powerlevels, nil
+}
+
 // LoadStateAtSnapshot loads the full state of a room at a particular snapshot.
 // This is typically the state before an event or the current state of a room.
 // Returns a sorted list of state entries or an error if there was a problem talking to the database.

roomserver/storage/interface.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/matrix-org/gomatrixserverlib"
 
+	"github.com/matrix-org/dendrite/roomserver/state"
 	"github.com/matrix-org/dendrite/roomserver/storage/shared"
 	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 	"github.com/matrix-org/dendrite/roomserver/types"
@@ -184,7 +185,7 @@ type Database interface {
 	GetOrCreateEventTypeNID(ctx context.Context, eventType string) (eventTypeNID types.EventTypeNID, err error)
 	GetOrCreateEventStateKeyNID(ctx context.Context, eventStateKey *string) (types.EventStateKeyNID, error)
 	MaybeRedactEvent(
-		ctx context.Context, roomInfo *types.RoomInfo, eventNID types.EventNID, event *gomatrixserverlib.Event, redactAllowed bool,
+		ctx context.Context, roomInfo *types.RoomInfo, eventNID types.EventNID, event *gomatrixserverlib.Event, plResolver state.PowerLevelResolver,
 	) (*gomatrixserverlib.Event, *gomatrixserverlib.Event, error)
 }
 
@@ -226,7 +227,7 @@ type EventDatabase interface {
 	// MaybeRedactEvent returns the redaction event and the redacted event if this call resulted in a redaction, else an error
 	// (nil if there was nothing to do)
 	MaybeRedactEvent(
-		ctx context.Context, roomInfo *types.RoomInfo, eventNID types.EventNID, event *gomatrixserverlib.Event, redactAllowed bool,
+		ctx context.Context, roomInfo *types.RoomInfo, eventNID types.EventNID, event *gomatrixserverlib.Event, plResolver state.PowerLevelResolver,
 	) (*gomatrixserverlib.Event, *gomatrixserverlib.Event, error)
 	StoreEvent(ctx context.Context, event *gomatrixserverlib.Event, roomInfo *types.RoomInfo, eventTypeNID types.EventTypeNID, eventStateKeyNID types.EventStateKeyNID, authEventNIDs []types.EventNID, isRejected bool) (types.EventNID, types.StateAtEvent, error)
 }

roomserver/storage/shared/storage.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/state"
 	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 	"github.com/matrix-org/dendrite/roomserver/types"
 )
@@ -660,6 +661,12 @@ func (d *Database) GetOrCreateRoomInfo(ctx context.Context, event *gomatrixserve
 	if roomVersion, err = extractRoomVersionFromCreateEvent(event); err != nil {
 		return nil, fmt.Errorf("extractRoomVersionFromCreateEvent: %w", err)
 	}
+	if roomVersion == "" {
+		rv, ok := d.Cache.GetRoomVersion(event.RoomID())
+		if ok {
+			roomVersion = rv
+		}
+	}
 	var roomNID types.RoomNID
 	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		roomNID, err = d.assignRoomNID(ctx, txn, event.RoomID(), roomVersion)
@@ -668,6 +675,9 @@ func (d *Database) GetOrCreateRoomInfo(ctx context.Context, event *gomatrixserve
 		}
 		return nil
 	})
+	if roomVersion != "" {
+		d.Cache.StoreRoomVersion(event.RoomID(), roomVersion)
+	}
 	return &types.RoomInfo{
 		RoomVersion: roomVersion,
 		RoomNID:     roomNID,
@@ -838,6 +848,7 @@ func (d *Database) assignRoomNID(
 		return 0, err
 	}
 	d.Cache.StoreRoomServerRoomID(roomNID, roomID)
+	d.Cache.StoreRoomVersion(roomID, roomVersion)
 	return roomNID, nil
 }
 
@@ -926,7 +937,7 @@ func extractRoomVersionFromCreateEvent(event *gomatrixserverlib.Event) (
 //
 // Returns the redaction event and the redacted event if this call resulted in a redaction.
 func (d *EventDatabase) MaybeRedactEvent(
-	ctx context.Context, roomInfo *types.RoomInfo, eventNID types.EventNID, event *gomatrixserverlib.Event, redactAllowed bool,
+	ctx context.Context, roomInfo *types.RoomInfo, eventNID types.EventNID, event *gomatrixserverlib.Event, plResolver state.PowerLevelResolver,
 ) (*gomatrixserverlib.Event, *gomatrixserverlib.Event, error) {
 	var (
 		redactionEvent, redactedEvent *types.Event
@@ -966,11 +977,20 @@ func (d *EventDatabase) MaybeRedactEvent(
 			return nil
 		}
 
-		// 1. The power level of the redaction event’s sender is greater than or equal to the redact level. (redactAllowed)
-		// 2. The domain of the redaction event’s sender matches that of the original event’s sender.
 		_, sender1, _ := gomatrixserverlib.SplitID('@', redactedEvent.Sender())
 		_, sender2, _ := gomatrixserverlib.SplitID('@', redactionEvent.Sender())
-		if !redactAllowed || sender1 != sender2 {
+		var powerlevels *gomatrixserverlib.PowerLevelContent
+		powerlevels, err = plResolver.Resolve(ctx, redactionEvent.EventID())
+		if err != nil {
+			return err
+		}
+
+		switch {
+		case powerlevels.UserLevel(redactionEvent.Sender()) >= powerlevels.Redact:
+			// 1. The power level of the redaction event’s sender is greater than or equal to the redact level.
+		case sender1 == sender2:
+			// 2. The domain of the redaction event’s sender matches that of the original event’s sender.
+		default:
 			ignoreRedaction = true
 			return nil
 		}
@@ -999,6 +1019,9 @@ func (d *EventDatabase) MaybeRedactEvent(
 		if err != nil {
 			return fmt.Errorf("d.RedactionsTable.MarkRedactionValidated: %w", err)
 		}
+
+		d.Cache.StoreRoomServerEvent(redactedEvent.EventNID, redactedEvent.Event)
+
 		return nil
 	})
 	if wErr != nil {