Merge branch 'master' into nats

build/gobind-pinecone/monolith.go

@@ -260,7 +260,7 @@ func (m *DendriteMonolith) Start() {
 
 	prefix := hex.EncodeToString(pk)
 	cfg := &config.Dendrite{}
-	cfg.Defaults()
+	cfg.Defaults(true)
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
 	cfg.Global.PrivateKey = sk
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)

build/gobind-yggdrasil/monolith.go

@@ -82,7 +82,7 @@ func (m *DendriteMonolith) Start() {
 	m.YggdrasilNode = ygg
 
 	cfg := &config.Dendrite{}
-	cfg.Defaults()
+	cfg.Defaults(true)
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
 	cfg.Global.PrivateKey = ygg.PrivateKey()
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)

clientapi/routing/register_test.go

@@ -180,7 +180,7 @@ func TestValidationOfApplicationServices(t *testing.T) {
 
 	// Set up a config
 	fakeConfig := &config.Dendrite{}
-	fakeConfig.Defaults()
+	fakeConfig.Defaults(true)
 	fakeConfig.Global.ServerName = "localhost"
 	fakeConfig.ClientAPI.Derived.ApplicationServices = []config.ApplicationService{fakeApplicationService}
 

cmd/dendrite-demo-libp2p/main.go

@@ -119,7 +119,7 @@ func main() {
 	}
 
 	cfg := config.Dendrite{}
-	cfg.Defaults()
+	cfg.Defaults(true)
 	cfg.Global.ServerName = "p2p"
 	cfg.Global.PrivateKey = privKey
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(fmt.Sprintf("ed25519:%s", *instanceName))

cmd/dendrite-demo-pinecone/main.go

@@ -142,7 +142,7 @@ func main() {
 	}
 
 	cfg := &config.Dendrite{}
-	cfg.Defaults()
+	cfg.Defaults(true)
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
 	cfg.Global.PrivateKey = sk
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)

cmd/dendrite-demo-yggdrasil/main.go

@@ -73,7 +73,7 @@ func main() {
 	*/
 
 	cfg := &config.Dendrite{}
-	cfg.Defaults()
+	cfg.Defaults(true)
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
 	cfg.Global.PrivateKey = ygg.PrivateKey()
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)

cmd/dendritejs-pinecone/main.go

@@ -161,7 +161,7 @@ func startup() {
 	pSessions := pineconeSessions.NewSessions(logger, pRouter)
 
 	cfg := &config.Dendrite{}
-	cfg.Defaults()
+	cfg.Defaults(true)
 	cfg.UserAPI.AccountDatabase.ConnectionString = "file:/idb/dendritejs_account.db"
 	cfg.AppServiceAPI.Database.ConnectionString = "file:/idb/dendritejs_appservice.db"
 	cfg.UserAPI.DeviceDatabase.ConnectionString = "file:/idb/dendritejs_device.db"

cmd/dendritejs/main.go

@@ -164,7 +164,7 @@ func createP2PNode(privKey ed25519.PrivateKey) (serverName string, node *go_http
 
 func main() {
 	cfg := &config.Dendrite{}
-	cfg.Defaults()
+	cfg.Defaults(true)
 	cfg.UserAPI.AccountDatabase.ConnectionString = "file:/idb/dendritejs_account.db"
 	cfg.AppServiceAPI.Database.ConnectionString = "file:/idb/dendritejs_appservice.db"
 	cfg.UserAPI.DeviceDatabase.ConnectionString = "file:/idb/dendritejs_device.db"

cmd/generate-config/main.go

@@ -17,7 +17,7 @@ func main() {
 	flag.Parse()
 
 	cfg := &config.Dendrite{}
-	cfg.Defaults()
+	cfg.Defaults(true)
 	if *serverName != "" {
 		cfg.Global.ServerName = gomatrixserverlib.ServerName(*serverName)
 	}

federationapi/federationapi_keys_test.go

@@ -71,7 +71,7 @@ func TestMain(m *testing.M) {
 		// Draw up just enough Dendrite config for the server key
 		// API to work.
 		cfg := &config.Dendrite{}
-		cfg.Defaults()
+		cfg.Defaults(true)
 		cfg.Global.ServerName = gomatrixserverlib.ServerName(s.name)
 		cfg.Global.PrivateKey = testPriv
 		cfg.Global.JetStream.InMemory = true

federationapi/federationapi_test.go

@@ -19,7 +19,7 @@ import (
 func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
 	_, privKey, _ := ed25519.GenerateKey(nil)
 	cfg := &config.Dendrite{}
-	cfg.Defaults()
+	cfg.Defaults(true)
 	cfg.Global.KeyID = gomatrixserverlib.KeyID("ed25519:auto")
 	cfg.Global.ServerName = gomatrixserverlib.ServerName("localhost")
 	cfg.Global.PrivateKey = privKey

internal/test/config.go

@@ -50,7 +50,7 @@ const (
 // Generates new matrix and TLS keys for the server.
 func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*config.Dendrite, int, error) {
 	var cfg config.Dendrite
-	cfg.Defaults()
+	cfg.Defaults(true)
 
 	port := startPort
 	assignAddress := func() config.HTTPAddress {

roomserver/roomserver_test.go

@@ -0,0 +1,407 @@
+package roomserver
+
+import (
+	"bytes"
+	"context"
+	"crypto/ed25519"
+	"encoding/json"
+	"fmt"
+	"os"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/Shopify/sarama"
+	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/roomserver/internal"
+	"github.com/matrix-org/dendrite/roomserver/storage"
+	"github.com/matrix-org/dendrite/setup/base"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	testOrigin = gomatrixserverlib.ServerName("kaer.morhen")
+	// we have to use an on-disk DB because we open multiple connections due to the *Updater structs.
+	// Using :memory: results in a brand new DB for each open connection, and sharing memory via
+	// ?cache=shared just allows read-only sharing, so writes to the database on other connections are lost.
+	roomserverDBFileURI  = "file:roomserver_test.db"
+	roomserverDBFilePath = "./roomserver_test.db"
+)
+
+var (
+	ctx = context.Background()
+)
+
+type dummyProducer struct {
+	topic            string
+	producedMessages []*api.OutputEvent
+}
+
+// SendMessage produces a given message, and returns only when it either has
+// succeeded or failed to produce. It will return the partition and the offset
+// of the produced message, or an error if the message failed to produce.
+func (p *dummyProducer) SendMessage(msg *sarama.ProducerMessage) (partition int32, offset int64, err error) {
+	if msg.Topic != p.topic {
+		return 0, 0, nil
+	}
+	be := msg.Value.(sarama.ByteEncoder)
+	b := json.RawMessage(be)
+	fmt.Println("SENDING >>>>>>>> ", string(b))
+	var out api.OutputEvent
+	err = json.Unmarshal(b, &out)
+	if err != nil {
+		return 0, 0, err
+	}
+	p.producedMessages = append(p.producedMessages, &out)
+	return 0, 0, nil
+}
+
+// SendMessages produces a given set of messages, and returns only when all
+// messages in the set have either succeeded or failed. Note that messages
+// can succeed and fail individually; if some succeed and some fail,
+// SendMessages will return an error.
+func (p *dummyProducer) SendMessages(msgs []*sarama.ProducerMessage) error {
+	for _, m := range msgs {
+		p.SendMessage(m)
+	}
+	return nil
+}
+
+// Close shuts down the producer and waits for any buffered messages to be
+// flushed. You must call this function before a producer object passes out of
+// scope, as it may otherwise leak memory. You must call this before calling
+// Close on the underlying client.
+func (p *dummyProducer) Close() error {
+	return nil
+}
+
+func deleteDatabase() {
+	err := os.Remove(roomserverDBFilePath)
+	if err != nil {
+		fmt.Printf("failed to delete database %s: %s\n", roomserverDBFilePath, err)
+	}
+}
+
+type fledglingEvent struct {
+	Type     string
+	StateKey *string
+	Content  interface{}
+	Sender   string
+	RoomID   string
+}
+
+func mustCreateEvents(t *testing.T, roomVer gomatrixserverlib.RoomVersion, events []fledglingEvent) (result []*gomatrixserverlib.HeaderedEvent) {
+	t.Helper()
+	depth := int64(1)
+	seed := make([]byte, ed25519.SeedSize) // zero seed
+	key := ed25519.NewKeyFromSeed(seed)
+	var prevs []string
+	roomState := make(map[gomatrixserverlib.StateKeyTuple]string) // state -> event ID
+	for _, ev := range events {
+		eb := gomatrixserverlib.EventBuilder{
+			Sender:     ev.Sender,
+			Depth:      depth,
+			Type:       ev.Type,
+			StateKey:   ev.StateKey,
+			RoomID:     ev.RoomID,
+			PrevEvents: prevs,
+		}
+		err := eb.SetContent(ev.Content)
+		if err != nil {
+			t.Fatalf("mustCreateEvent: failed to marshal event content %+v", ev.Content)
+		}
+		stateNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(&eb)
+		if err != nil {
+			t.Fatalf("mustCreateEvent: failed to work out auth_events : %s", err)
+		}
+		var authEvents []string
+		for _, tuple := range stateNeeded.Tuples() {
+			eventID := roomState[tuple]
+			if eventID != "" {
+				authEvents = append(authEvents, eventID)
+			}
+		}
+		eb.AuthEvents = authEvents
+		signedEvent, err := eb.Build(time.Now(), testOrigin, "ed25519:test", key, roomVer)
+		if err != nil {
+			t.Fatalf("mustCreateEvent: failed to sign event: %s", err)
+		}
+		depth++
+		prevs = []string{signedEvent.EventID()}
+		if ev.StateKey != nil {
+			roomState[gomatrixserverlib.StateKeyTuple{
+				EventType: ev.Type,
+				StateKey:  *ev.StateKey,
+			}] = signedEvent.EventID()
+		}
+		result = append(result, signedEvent.Headered(roomVer))
+	}
+	return
+}
+
+func mustLoadRawEvents(t *testing.T, ver gomatrixserverlib.RoomVersion, events []json.RawMessage) []*gomatrixserverlib.HeaderedEvent {
+	t.Helper()
+	hs := make([]*gomatrixserverlib.HeaderedEvent, len(events))
+	for i := range events {
+		e, err := gomatrixserverlib.NewEventFromTrustedJSON(events[i], false, ver)
+		if err != nil {
+			t.Fatalf("cannot load test data: " + err.Error())
+		}
+		hs[i] = e.Headered(ver)
+	}
+	return hs
+}
+
+func mustCreateRoomserverAPI(t *testing.T) (api.RoomserverInternalAPI, *dummyProducer) {
+	t.Helper()
+	cfg := &config.Dendrite{}
+	cfg.Defaults(true)
+	cfg.Global.ServerName = testOrigin
+	cfg.Global.Kafka.UseNaffka = true
+	cfg.RoomServer.Database = config.DatabaseOptions{
+		ConnectionString: roomserverDBFileURI,
+	}
+	dp := &dummyProducer{
+		topic: cfg.Global.Kafka.TopicFor(config.TopicOutputRoomEvent),
+	}
+	cache, err := caching.NewInMemoryLRUCache(false)
+	if err != nil {
+		t.Fatalf("failed to make caches: %s", err)
+	}
+	base := &base.BaseDendrite{
+		Caches: cache,
+		Cfg:    cfg,
+	}
+	roomserverDB, err := storage.Open(&cfg.RoomServer.Database, base.Caches)
+	if err != nil {
+		logrus.WithError(err).Panicf("failed to connect to room server db")
+	}
+	return internal.NewRoomserverAPI(
+		&cfg.RoomServer, roomserverDB, dp, string(cfg.Global.Kafka.TopicFor(config.TopicOutputRoomEvent)),
+		base.Caches, nil,
+	), dp
+}
+
+func mustSendEvents(t *testing.T, ver gomatrixserverlib.RoomVersion, events []json.RawMessage) (api.RoomserverInternalAPI, *dummyProducer, []*gomatrixserverlib.HeaderedEvent) {
+	t.Helper()
+	rsAPI, dp := mustCreateRoomserverAPI(t)
+	hevents := mustLoadRawEvents(t, ver, events)
+	if err := api.SendEvents(ctx, rsAPI, api.KindNew, hevents, testOrigin, nil); err != nil {
+		t.Errorf("failed to SendEvents: %s", err)
+	}
+	return rsAPI, dp, hevents
+}
+
+func TestOutputRedactedEvent(t *testing.T) {
+	redactionEvents := []json.RawMessage{
+		// create event
+		[]byte(`{"auth_events":[],"content":{"creator":"@userid:kaer.morhen"},"depth":0,"event_id":"$N4us6vqqq3RjvpKd:kaer.morhen","hashes":{"sha256":"WTdrCn/YsiounXcJPsLP8xT0ZjHiO5Ov0NvXYmK2onE"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"9+5JcpaN5b5KlHYHGp6r+GoNDH98lbfzGYwjfxensa5C5D/bDACaYnMDLnhwsHOE5nxgI+jT/GV271pz6PMSBQ"}},"state_key":"","type":"m.room.create"}`),
+		// join event
+		[]byte(`{"auth_events":[["$N4us6vqqq3RjvpKd:kaer.morhen",{"sha256":"SylirfgfXFhscZL7p10NmOa1nFFEckiwz0lAideQMIM"}]],"content":{"membership":"join"},"depth":1,"event_id":"$6sUiGPQ0a3tqYGKo:kaer.morhen","hashes":{"sha256":"eYVBC7RO+FlxRyW1aXYf/ad4Dzi7T93tArdGw3r4RwQ"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$N4us6vqqq3RjvpKd:kaer.morhen",{"sha256":"SylirfgfXFhscZL7p10NmOa1nFFEckiwz0lAideQMIM"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"tiDBTPFa53YMfHiupX3vSRE/ZcCiCjmGt7gDpIpDpwZapeays5Vqqcqb7KiywrDldpTkrrdJBAw2jXcq6ZyhDw"}},"state_key":"@userid:kaer.morhen","type":"m.room.member"}`),
+		// room name
+		[]byte(`{"auth_events":[["$N4us6vqqq3RjvpKd:kaer.morhen",{"sha256":"SylirfgfXFhscZL7p10NmOa1nFFEckiwz0lAideQMIM"}],["$6sUiGPQ0a3tqYGKo:kaer.morhen",{"sha256":"IS4HSMqpqVUGh1Z3qgC99YcaizjCoO4yFhYYe8j53IE"}]],"content":{"name":"My Room Name"},"depth":2,"event_id":"$VC1zZ9YWwuUbSNHD:kaer.morhen","hashes":{"sha256":"bpqTkfLx6KHzWz7/wwpsXnXwJWEGW14aV63ffexzDFg"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$6sUiGPQ0a3tqYGKo:kaer.morhen",{"sha256":"IS4HSMqpqVUGh1Z3qgC99YcaizjCoO4yFhYYe8j53IE"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"mhJZ3X4bAKrF/T0mtPf1K2Tmls0h6xGY1IPDpJ/SScQBqDlu3HQR2BPa7emqj5bViyLTWVNh+ZCpzx/6STTrAg"}},"state_key":"","type":"m.room.name"}`),
+		// redact room name
+		[]byte(`{"auth_events":[["$N4us6vqqq3RjvpKd:kaer.morhen",{"sha256":"SylirfgfXFhscZL7p10NmOa1nFFEckiwz0lAideQMIM"}],["$6sUiGPQ0a3tqYGKo:kaer.morhen",{"sha256":"IS4HSMqpqVUGh1Z3qgC99YcaizjCoO4yFhYYe8j53IE"}]],"content":{"reason":"Spamming"},"depth":3,"event_id":"$tJI0pE3b8u9UMYpT:kaer.morhen","hashes":{"sha256":"/3TStqa5SQqYaEtl7ajEvSRvu6d12MMKfICUzrBpd2Q"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$VC1zZ9YWwuUbSNHD:kaer.morhen",{"sha256":"+l8cNa7syvm0EF7CAmQRlYknLEMjivnI4FLhB/TUBEY"}]],"redacts":"$VC1zZ9YWwuUbSNHD:kaer.morhen","room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"QBOh+amf0vTJbm6+9VwAcR9uJviBIor2KON0Y7+EyQx5YbUZEzW1HPeJxarLIHBcxMzgOVzjuM+StzjbUgDzAg"}},"type":"m.room.redaction"}`),
+		// message
+		[]byte(`{"auth_events":[["$N4us6vqqq3RjvpKd:kaer.morhen",{"sha256":"SylirfgfXFhscZL7p10NmOa1nFFEckiwz0lAideQMIM"}],["$6sUiGPQ0a3tqYGKo:kaer.morhen",{"sha256":"IS4HSMqpqVUGh1Z3qgC99YcaizjCoO4yFhYYe8j53IE"}]],"content":{"body":"Test Message"},"depth":4,"event_id":"$o8KHsgSIYbJrddnd:kaer.morhen","hashes":{"sha256":"IE/rGVlKOpiGWeIo887g1CK1drYqcWDZhL6THZHkJ1c"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$tJI0pE3b8u9UMYpT:kaer.morhen",{"sha256":"zvmwyXuDox7jpA16JRH6Fc1zbfQht2tpkBbMTUOi3Jw"}]],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"/3z+pJjiJXWhwfqIEzmNksvBHCoXTktK/y0rRuWJXw6i1+ygRG/suDCKhFuuz6gPapRmEMPVILi2mJqHHXPKAg"}},"type":"m.room.message"}`),
+		// redact previous message
+		[]byte(`{"auth_events":[["$N4us6vqqq3RjvpKd:kaer.morhen",{"sha256":"SylirfgfXFhscZL7p10NmOa1nFFEckiwz0lAideQMIM"}],["$6sUiGPQ0a3tqYGKo:kaer.morhen",{"sha256":"IS4HSMqpqVUGh1Z3qgC99YcaizjCoO4yFhYYe8j53IE"}]],"content":{"reason":"Spamming more"},"depth":5,"event_id":"$UpsE8belb2gJItJG:kaer.morhen","hashes":{"sha256":"zU8PWJOld/I7OtjdpltFSKC+DMNm2ZyEXAHcprsafD0"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$o8KHsgSIYbJrddnd:kaer.morhen",{"sha256":"UgjMuCFXH4warIjKuwlRq9zZ6dSJrZWCd+CkqtgLSHM"}]],"redacts":"$o8KHsgSIYbJrddnd:kaer.morhen","room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"zxFGr/7aGOzqOEN6zRNrBpFkkMnfGFPbCteYL33wC+PycBPIK+2WRa5qlAR2+lcLiK3HjIzwRYkKNsVFTqvRAw"}},"type":"m.room.redaction"}`),
+	}
+	var redactedOutputs []api.OutputEvent
+	deleteDatabase()
+	_, producer, hevents := mustSendEvents(t, gomatrixserverlib.RoomVersionV1, redactionEvents)
+	defer deleteDatabase()
+	for _, msg := range producer.producedMessages {
+		if msg.Type == api.OutputTypeRedactedEvent {
+			redactedOutputs = append(redactedOutputs, *msg)
+		}
+	}
+	wantRedactedOutputs := []api.OutputEvent{
+		{
+			Type: api.OutputTypeRedactedEvent,
+			RedactedEvent: &api.OutputRedactedEvent{
+				RedactedEventID: hevents[2].EventID(),
+				RedactedBecause: hevents[3],
+			},
+		},
+		{
+			Type: api.OutputTypeRedactedEvent,
+			RedactedEvent: &api.OutputRedactedEvent{
+				RedactedEventID: hevents[4].EventID(),
+				RedactedBecause: hevents[5],
+			},
+		},
+	}
+	t.Logf("redactedOutputs: %+v", redactedOutputs)
+	if len(wantRedactedOutputs) != len(redactedOutputs) {
+		t.Fatalf("Got %d redacted events, want %d", len(redactedOutputs), len(wantRedactedOutputs))
+	}
+	for i := 0; i < len(wantRedactedOutputs); i++ {
+		if !reflect.DeepEqual(*redactedOutputs[i].RedactedEvent, *wantRedactedOutputs[i].RedactedEvent) {
+			t.Errorf("OutputRedactionEvent %d: wrong event got:\n%+v want:\n%+v", i+1, redactedOutputs[i].RedactedEvent, wantRedactedOutputs[i].RedactedEvent)
+		}
+	}
+}
+
+// This tests that rewriting state works correctly.
+// This creates a small room with a create/join/name state, then replays it
+// with a new room name. We expect the output events to contain the original events,
+// followed by a single OutputNewRoomEvent with RewritesState set to true with the
+// rewritten state events (with the 2nd room name).
+func TestOutputRewritesState(t *testing.T) {
+	roomID := "!foo:" + string(testOrigin)
+	alice := "@alice:" + string(testOrigin)
+	emptyKey := ""
+	originalEvents := mustCreateEvents(t, gomatrixserverlib.RoomVersionV6, []fledglingEvent{
+		{
+			RoomID: roomID,
+			Sender: alice,
+			Content: map[string]interface{}{
+				"creator":      alice,
+				"room_version": "6",
+			},
+			StateKey: &emptyKey,
+			Type:     gomatrixserverlib.MRoomCreate,
+		},
+		{
+			RoomID: roomID,
+			Sender: alice,
+			Content: map[string]interface{}{
+				"membership": "join",
+			},
+			StateKey: &alice,
+			Type:     gomatrixserverlib.MRoomMember,
+		},
+		{
+			RoomID: roomID,
+			Sender: alice,
+			Content: map[string]interface{}{
+				"body": "hello world",
+			},
+			StateKey: nil,
+			Type:     "m.room.message",
+		},
+		{
+			RoomID: roomID,
+			Sender: alice,
+			Content: map[string]interface{}{
+				"name": "Room Name",
+			},
+			StateKey: &emptyKey,
+			Type:     "m.room.name",
+		},
+	})
+	rewriteEvents := mustCreateEvents(t, gomatrixserverlib.RoomVersionV6, []fledglingEvent{
+		{
+			RoomID: roomID,
+			Sender: alice,
+			Content: map[string]interface{}{
+				"creator": alice,
+			},
+			StateKey: &emptyKey,
+			Type:     gomatrixserverlib.MRoomCreate,
+		},
+		{
+			RoomID: roomID,
+			Sender: alice,
+			Content: map[string]interface{}{
+				"membership": "join",
+			},
+			StateKey: &alice,
+			Type:     gomatrixserverlib.MRoomMember,
+		},
+		{
+			RoomID: roomID,
+			Sender: alice,
+			Content: map[string]interface{}{
+				"name": "Room Name 2",
+			},
+			StateKey: &emptyKey,
+			Type:     "m.room.name",
+		},
+		{
+			RoomID: roomID,
+			Sender: alice,
+			Content: map[string]interface{}{
+				"body": "hello world 2",
+			},
+			StateKey: nil,
+			Type:     "m.room.message",
+		},
+	})
+	deleteDatabase()
+	rsAPI, producer := mustCreateRoomserverAPI(t)
+	defer deleteDatabase()
+	err := api.SendEvents(context.Background(), rsAPI, api.KindNew, originalEvents, testOrigin, nil)
+	if err != nil {
+		t.Fatalf("failed to send original events: %s", err)
+	}
+	// assert we got them produced, this is just a sanity check and isn't the intention of this test
+	if len(producer.producedMessages) != len(originalEvents) {
+		t.Fatalf("SendEvents didn't result in same number of produced output events: got %d want %d", len(producer.producedMessages), len(originalEvents))
+	}
+	producer.producedMessages = nil // we aren't actually interested in these events, just the rewrite ones
+
+	var inputEvents []api.InputRoomEvent
+	// slowly build up the state IDs again, we're basically telling the roomserver what to store as a snapshot
+	var stateIDs []string
+	// skip the last event, we'll use this to tie together the rewrite as the KindNew event
+	for i := 0; i < len(rewriteEvents)-1; i++ {
+		ev := rewriteEvents[i]
+		inputEvents = append(inputEvents, api.InputRoomEvent{
+			Kind:          api.KindOutlier,
+			Event:         ev,
+			AuthEventIDs:  ev.AuthEventIDs(),
+			HasState:      true,
+			StateEventIDs: stateIDs,
+		})
+		if ev.StateKey() != nil {
+			stateIDs = append(stateIDs, ev.EventID())
+		}
+	}
+	lastEv := rewriteEvents[len(rewriteEvents)-1]
+	inputEvents = append(inputEvents, api.InputRoomEvent{
+		Kind:          api.KindNew,
+		Event:         lastEv,
+		AuthEventIDs:  lastEv.AuthEventIDs(),
+		HasState:      true,
+		StateEventIDs: stateIDs,
+	})
+	if err := api.SendInputRoomEvents(context.Background(), rsAPI, inputEvents); err != nil {
+		t.Fatalf("SendInputRoomEvents returned error for rewrite events: %s", err)
+	}
+	// we should just have one output event with the entire state of the room in it
+	if len(producer.producedMessages) != 1 {
+		t.Fatalf("Rewritten events got output, want only 1 got %d", len(producer.producedMessages))
+	}
+	outputEvent := producer.producedMessages[len(producer.producedMessages)-1]
+	if !outputEvent.NewRoomEvent.RewritesState {
+		t.Errorf("RewritesState flag not set on output event")
+	}
+	if !reflect.DeepEqual(stateIDs, outputEvent.NewRoomEvent.AddsStateEventIDs) {
+		t.Errorf("Output event is missing room state event IDs, got %v want %v", outputEvent.NewRoomEvent.AddsStateEventIDs, stateIDs)
+	}
+	if !bytes.Equal(outputEvent.NewRoomEvent.Event.JSON(), lastEv.JSON()) {
+		t.Errorf(
+			"Output event isn't the latest KindNew event:\ngot  %s\nwant %s",
+			string(outputEvent.NewRoomEvent.Event.JSON()),
+			string(lastEv.JSON()),
+		)
+	}
+	if len(outputEvent.NewRoomEvent.AddStateEvents) != len(stateIDs) {
+		t.Errorf("Output event is missing room state events themselves, got %d want %d", len(outputEvent.NewRoomEvent.AddStateEvents), len(stateIDs))
+	}
+	// make sure the state got overwritten, check the room name
+	hasRoomName := false
+	for _, ev := range outputEvent.NewRoomEvent.AddStateEvents {
+		if ev.Type() == "m.room.name" {
+			hasRoomName = string(ev.Content()) == `{"name":"Room Name 2"}`
+		}
+	}
+	if !hasRoomName {
+		t.Errorf("Output event did not overwrite room state")
+	}
+}

setup/config/config.go

@@ -210,7 +210,7 @@ func loadConfig(
 	monolithic bool,
 ) (*Dendrite, error) {
 	var c Dendrite
-	c.Defaults()
+	c.Defaults(false)
 
 	var err error
 	if err = yaml.Unmarshal(configData, &c); err != nil {
@@ -291,22 +291,22 @@ func (config *Dendrite) Derive() error {
 }
 
 // SetDefaults sets default config values if they are not explicitly set.
-func (c *Dendrite) Defaults() {
+func (c *Dendrite) Defaults(generate bool) {
 	c.Version = 1
 
-	c.Wiring()
+	c.Global.Defaults(generate)
+	c.ClientAPI.Defaults(generate)
+	c.EDUServer.Defaults(generate)
+	c.FederationAPI.Defaults(generate)
+	c.KeyServer.Defaults(generate)
+	c.MediaAPI.Defaults(generate)
+	c.RoomServer.Defaults(generate)
+	c.SyncAPI.Defaults(generate)
+	c.UserAPI.Defaults(generate)
+	c.AppServiceAPI.Defaults(generate)
+	c.MSCs.Defaults(generate)
 
-	c.Global.Defaults()
+	c.Wiring()
-	c.ClientAPI.Defaults()
-	c.EDUServer.Defaults()
-	c.FederationAPI.Defaults()
-	c.KeyServer.Defaults()
-	c.MediaAPI.Defaults()
-	c.RoomServer.Defaults()
-	c.SyncAPI.Defaults()
-	c.UserAPI.Defaults()
-	c.AppServiceAPI.Defaults()
-	c.MSCs.Defaults()
 }
 
 func (c *Dendrite) Verify(configErrs *ConfigErrors, isMonolith bool) {

setup/config/config_appservice.go

@@ -40,12 +40,14 @@ type AppServiceAPI struct {
 	ConfigFiles []string `yaml:"config_files"`
 }
 
-func (c *AppServiceAPI) Defaults() {
+func (c *AppServiceAPI) Defaults(generate bool) {
 	c.InternalAPI.Listen = "http://localhost:7777"
 	c.InternalAPI.Connect = "http://localhost:7777"
 	c.Database.Defaults(5)
+	if generate {
 		c.Database.ConnectionString = "file:appservice.db"
 	}
+}
 
 func (c *AppServiceAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkURL(configErrs, "app_service_api.internal_api.listen", string(c.InternalAPI.Listen))

setup/config/config_clientapi.go

@@ -41,7 +41,7 @@ type ClientAPI struct {
 	MSCs *MSCs `yaml:"mscs"`
 }
 
-func (c *ClientAPI) Defaults() {
+func (c *ClientAPI) Defaults(generate bool) {
 	c.InternalAPI.Listen = "http://localhost:7771"
 	c.InternalAPI.Connect = "http://localhost:7771"
 	c.ExternalAPI.Listen = "http://[::]:8071"

setup/config/config_eduserver.go

@@ -6,7 +6,7 @@ type EDUServer struct {
 	InternalAPI InternalAPIOptions `yaml:"internal_api"`
 }
 
-func (c *EDUServer) Defaults() {
+func (c *EDUServer) Defaults(generate bool) {
 	c.InternalAPI.Listen = "http://localhost:7778"
 	c.InternalAPI.Connect = "http://localhost:7778"
 }

setup/config/config_federationapi.go

@@ -39,12 +39,14 @@ type FederationAPI struct {
 	PreferDirectFetch bool `yaml:"prefer_direct_fetch"`
 }
 
-func (c *FederationAPI) Defaults() {
+func (c *FederationAPI) Defaults(generate bool) {
 	c.InternalAPI.Listen = "http://localhost:7772"
 	c.InternalAPI.Connect = "http://localhost:7772"
 	c.ExternalAPI.Listen = "http://[::]:8072"
 	c.Database.Defaults(10)
+	if generate {
 		c.Database.ConnectionString = "file:federationapi.db"
+	}
 
 	c.FederationMaxRetries = 16
 	c.DisableTLSValidation = false

setup/config/config_global.go

@@ -59,15 +59,17 @@ type Global struct {
 	DNSCache DNSCacheOptions `yaml:"dns_cache"`
 }
 
-func (c *Global) Defaults() {
+func (c *Global) Defaults(generate bool) {
+	if generate {
 		c.ServerName = "localhost"
 		c.PrivateKeyPath = "matrix_key.pem"
 		_, c.PrivateKey, _ = ed25519.GenerateKey(rand.New(rand.NewSource(0)))
 		c.KeyID = "ed25519:auto"
+	}
 	c.KeyValidityPeriod = time.Hour * 24 * 7
 
-	c.JetStream.Defaults()
+	c.JetStream.Defaults(generate)
-	c.Metrics.Defaults()
+	c.Metrics.Defaults(generate)
 	c.DNSCache.Defaults()
 	c.Sentry.Defaults()
 }
@@ -110,11 +112,13 @@ type Metrics struct {
 	} `yaml:"basic_auth"`
 }
 
-func (c *Metrics) Defaults() {
+func (c *Metrics) Defaults(generate bool) {
 	c.Enabled = false
+	if generate {
 		c.BasicAuth.Username = "metrics"
 		c.BasicAuth.Password = "metrics"
 	}
+}
 
 func (c *Metrics) Verify(configErrs *ConfigErrors, isMonolith bool) {
 }

setup/config/config_jetstream.go

@@ -23,11 +23,13 @@ func (c *JetStream) TopicFor(name string) string {
 	return fmt.Sprintf("%s%s", c.TopicPrefix, name)
 }
 
-func (c *JetStream) Defaults() {
+func (c *JetStream) Defaults(generate bool) {
 	c.Addresses = []string{}
 	c.TopicPrefix = "Dendrite"
+	if generate {
 		c.StoragePath = Path("./")
 	}
+}
 
 func (c *JetStream) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	// If we are running in a polylith deployment then we need at least

setup/config/config_keyserver.go

@@ -8,12 +8,14 @@ type KeyServer struct {
 	Database DatabaseOptions `yaml:"database"`
 }
 
-func (c *KeyServer) Defaults() {
+func (c *KeyServer) Defaults(generate bool) {
 	c.InternalAPI.Listen = "http://localhost:7779"
 	c.InternalAPI.Connect = "http://localhost:7779"
 	c.Database.Defaults(10)
+	if generate {
 		c.Database.ConnectionString = "file:keyserver.db"
 	}
+}
 
 func (c *KeyServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkURL(configErrs, "key_server.internal_api.listen", string(c.InternalAPI.Listen))

setup/config/config_mediaapi.go

@@ -38,16 +38,18 @@ type MediaAPI struct {
 // DefaultMaxFileSizeBytes defines the default file size allowed in transfers
 var DefaultMaxFileSizeBytes = FileSizeBytes(10485760)
 
-func (c *MediaAPI) Defaults() {
+func (c *MediaAPI) Defaults(generate bool) {
 	c.InternalAPI.Listen = "http://localhost:7774"
 	c.InternalAPI.Connect = "http://localhost:7774"
 	c.ExternalAPI.Listen = "http://[::]:8074"
 	c.Database.Defaults(5)
+	if generate {
 		c.Database.ConnectionString = "file:mediaapi.db"
+		c.BasePath = "./media_store"
+	}
 
 	c.MaxFileSizeBytes = &DefaultMaxFileSizeBytes
 	c.MaxThumbnailGenerators = 10
-	c.BasePath = "./media_store"
 }
 
 func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {

setup/config/config_mscs.go

@@ -13,10 +13,12 @@ type MSCs struct {
 	Database DatabaseOptions `yaml:"database"`
 }
 
-func (c *MSCs) Defaults() {
+func (c *MSCs) Defaults(generate bool) {
 	c.Database.Defaults(5)
+	if generate {
 		c.Database.ConnectionString = "file:mscs.db"
 	}
+}
 
 // Enabled returns true if the given msc is enabled. Should in the form 'msc12345'.
 func (c *MSCs) Enabled(msc string) bool {

setup/config/config_roomserver.go

@@ -8,12 +8,14 @@ type RoomServer struct {
 	Database DatabaseOptions `yaml:"database"`
 }
 
-func (c *RoomServer) Defaults() {
+func (c *RoomServer) Defaults(generate bool) {
 	c.InternalAPI.Listen = "http://localhost:7770"
 	c.InternalAPI.Connect = "http://localhost:7770"
 	c.Database.Defaults(10)
+	if generate {
 		c.Database.ConnectionString = "file:roomserver.db"
 	}
+}
 
 func (c *RoomServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkURL(configErrs, "room_server.internal_api.listen", string(c.InternalAPI.Listen))

setup/config/config_syncapi.go

@@ -11,13 +11,15 @@ type SyncAPI struct {
 	RealIPHeader string `yaml:"real_ip_header"`
 }
 
-func (c *SyncAPI) Defaults() {
+func (c *SyncAPI) Defaults(generate bool) {
 	c.InternalAPI.Listen = "http://localhost:7773"
 	c.InternalAPI.Connect = "http://localhost:7773"
 	c.ExternalAPI.Listen = "http://localhost:8073"
 	c.Database.Defaults(10)
+	if generate {
 		c.Database.ConnectionString = "file:syncapi.db"
 	}
+}
 
 func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkURL(configErrs, "sync_api.internal_api.listen", string(c.InternalAPI.Listen))

setup/config/config_userapi.go

@@ -23,13 +23,15 @@ type UserAPI struct {
 
 const DefaultOpenIDTokenLifetimeMS = 3600000 // 60 minutes
 
-func (c *UserAPI) Defaults() {
+func (c *UserAPI) Defaults(generate bool) {
 	c.InternalAPI.Listen = "http://localhost:7781"
 	c.InternalAPI.Connect = "http://localhost:7781"
 	c.AccountDatabase.Defaults(10)
 	c.DeviceDatabase.Defaults(10)
+	if generate {
 		c.AccountDatabase.ConnectionString = "file:userapi_accounts.db"
 		c.DeviceDatabase.ConnectionString = "file:userapi_devices.db"
+	}
 	c.BCryptCost = bcrypt.DefaultCost
 	c.OpenIDTokenLifetimeMS = DefaultOpenIDTokenLifetimeMS
 }

setup/mscs/msc2836/msc2836_test.go

@@ -545,7 +545,7 @@ func (r *testRoomserverAPI) QueryMembershipForUser(ctx context.Context, req *roo
 func injectEvents(t *testing.T, userAPI userapi.UserInternalAPI, rsAPI roomserver.RoomserverInternalAPI, events []*gomatrixserverlib.HeaderedEvent) *mux.Router {
 	t.Helper()
 	cfg := &config.Dendrite{}
-	cfg.Defaults()
+	cfg.Defaults(true)
 	cfg.Global.ServerName = "localhost"
 	cfg.MSCs.Database.ConnectionString = "file:msc2836_test.db"
 	cfg.MSCs.MSCs = []string{"msc2836"}

setup/mscs/msc2946/msc2946_test.go

@@ -411,7 +411,7 @@ func (r *testRoomserverAPI) QueryCurrentState(ctx context.Context, req *roomserv
 func injectEvents(t *testing.T, userAPI userapi.UserInternalAPI, rsAPI roomserver.RoomserverInternalAPI, events []*gomatrixserverlib.HeaderedEvent) *mux.Router {
 	t.Helper()
 	cfg := &config.Dendrite{}
-	cfg.Defaults()
+	cfg.Defaults(true)
 	cfg.Global.ServerName = "localhost"
 	cfg.MSCs.Database.ConnectionString = "file:msc2946_test.db"
 	cfg.MSCs.MSCs = []string{"msc2946"}