Merge branch 'master' into master

.github/workflows/docker-hub.yml

@@ -0,0 +1,70 @@
+# Based on https://github.com/docker/build-push-action
+
+name: "Docker Hub"
+
+on:
+  release:
+    types: [published]
+
+env:
+  DOCKER_HUB_USER: matrixdotorg
+  PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7
+
+jobs:
+  Monolith:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Get release tag
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1 
+        with:
+          username: ${{ env.DOCKER_HUB_USER }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Build monolith image
+        id: docker_build_monolith
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./build/docker/Dockerfile.monolith
+          platforms: ${{ env.PLATFORMS }}
+          push: true
+          tags: |
+            ${{ env.DOCKER_HUB_USER }}/dendrite-monolith:latest
+            ${{ env.DOCKER_HUB_USER }}/dendrite-monolith:${{ env.RELEASE_VERSION }}
+
+  Polylith:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Get release tag
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1 
+        with:
+          username: ${{ env.DOCKER_HUB_USER }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Build polylith image
+        id: docker_build_polylith
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./build/docker/Dockerfile.polylith
+          platforms: ${{ env.PLATFORMS }}
+          push: true
+          tags: |
+            ${{ env.DOCKER_HUB_USER }}/dendrite-polylith:latest
+            ${{ env.DOCKER_HUB_USER }}/dendrite-polylith:${{ env.RELEASE_VERSION }}

build.sh

@@ -17,6 +17,6 @@ else
     export FLAGS=""
 fi
 
-go install -trimpath -ldflags "$FLAGS" -v $PWD/`dirname $0`/cmd/...
+CGO_ENABLED=1 go build -trimpath -ldflags "$FLAGS" -v -o "bin/" ./cmd/...
 
-GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs
+CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs

build/docker/Dockerfile

@@ -1,10 +0,0 @@
-FROM docker.io/golang:1.15-alpine AS builder
-
-RUN apk --update --no-cache add bash build-base
-
-WORKDIR /build
-
-COPY . /build
-
-RUN mkdir -p bin
-RUN sh ./build.sh

build/docker/Dockerfile.monolith

@@ -1,11 +1,20 @@
-FROM matrixdotorg/dendrite:latest AS base
+FROM docker.io/golang:1.15-alpine AS base
+
+RUN apk --update --no-cache add bash build-base
+
+WORKDIR /build
+
+COPY . /build
+
+RUN mkdir -p bin
+RUN go build -trimpath -o bin/ ./cmd/dendrite-monolith-server
+RUN go build -trimpath -o bin/ ./cmd/goose
+RUN go build -trimpath -o bin/ ./cmd/create-account
+RUN go build -trimpath -o bin/ ./cmd/generate-keys
 
 FROM alpine:latest
 
-COPY --from=base /build/bin/dendrite-monolith-server /usr/bin
-COPY --from=base /build/bin/goose /usr/bin
-COPY --from=base /build/bin/create-account /usr/bin
-COPY --from=base /build/bin/generate-keys /usr/bin
+COPY --from=base /build/bin/* /usr/bin
 
 VOLUME /etc/dendrite
 WORKDIR /etc/dendrite

build/docker/Dockerfile.polylith

@@ -1,11 +1,20 @@
-FROM matrixdotorg/dendrite:latest AS base
+FROM docker.io/golang:1.15-alpine AS base
+
+RUN apk --update --no-cache add bash build-base
+
+WORKDIR /build
+
+COPY . /build
+
+RUN mkdir -p bin
+RUN go build -trimpath -o bin/ ./cmd/dendrite-polylith-multi
+RUN go build -trimpath -o bin/ ./cmd/goose
+RUN go build -trimpath -o bin/ ./cmd/create-account
+RUN go build -trimpath -o bin/ ./cmd/generate-keys
 
 FROM alpine:latest
 
-COPY --from=base /build/bin/dendrite-polylith-multi /usr/bin
-COPY --from=base /build/bin/goose /usr/bin
-COPY --from=base /build/bin/create-account /usr/bin
-COPY --from=base /build/bin/generate-keys /usr/bin
+COPY --from=base /build/bin/* /usr/bin
 
 VOLUME /etc/dendrite
 WORKDIR /etc/dendrite

build/docker/images-build.sh

@@ -6,7 +6,5 @@ TAG=${1:-latest}
 
 echo "Building tag '${TAG}'"
 
-docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:${TAG} .
-
 docker build -t matrixdotorg/dendrite-monolith:${TAG}  -f build/docker/Dockerfile.monolith .
 docker build -t matrixdotorg/dendrite-polylith:${TAG}  -f build/docker/Dockerfile.polylith .

cmd/generate-config/main.go

@@ -63,7 +63,7 @@ func main() {
 	if *defaultsForCI {
 		cfg.ClientAPI.RateLimiting.Enabled = false
 		cfg.FederationSender.DisableTLSValidation = true
-		cfg.MSCs.MSCs = []string{"msc2836"}
+		cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
 		cfg.Logging[0].Level = "trace"
 		// don't hit matrix.org when running tests!!!
 		cfg.SigningKeyServer.KeyPerspectives = config.KeyPerspectives{}

keyserver/internal/device_list_update.go

@@ -245,7 +245,7 @@ func (u *DeviceListUpdater) notifyWorkers(userID string) {
 	}
 	hash := fnv.New32a()
 	_, _ = hash.Write([]byte(remoteServer))
-	index := int(hash.Sum32()) % len(u.workerChans)
+	index := int(int64(hash.Sum32()) % int64(len(u.workerChans)))
 
 	ch := u.assignChannel(userID)
 	u.workerChans[index] <- remoteServer

setup/mscs/msc2946/msc2946.go

@@ -19,6 +19,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"strings"
 	"sync"
 
 	"github.com/gorilla/mux"
@@ -36,7 +37,7 @@ import (
 const (
 	ConstCreateEventContentKey = "org.matrix.msc1772.type"
 	ConstSpaceChildEventType   = "org.matrix.msc1772.space.child"
-	ConstSpaceParentEventType  = "org.matrix.msc1772.room.parent"
+	ConstSpaceParentEventType  = "org.matrix.msc1772.space.parent"
 )
 
 // SpacesRequest is the request body to POST /_matrix/client/r0/rooms/{roomID}/spaces
@@ -57,7 +58,7 @@ type SpacesResponse struct {
 	NextBatch string `json:"next_batch"`
 	// Rooms are nodes on the space graph.
 	Rooms []Room `json:"rooms"`
-	// Events are edges on the space graph, exclusively m.space.child or m.room.parent events
+	// Events are edges on the space graph, exclusively m.space.child or m.space.parent events
 	Events []gomatrixserverlib.ClientEvent `json:"events"`
 }
 
@@ -182,8 +183,8 @@ func (w *walker) walk() *SpacesResponse {
 		if !w.authorised(roomID) {
 			continue
 		}
-		// Get all `m.space.child` and `m.room.parent` state events for the room. *In addition*, get
-		// all `m.space.child` and `m.room.parent` state events which *point to* (via `state_key` or `content.room_id`)
+		// Get all `m.space.child` and `m.space.parent` state events for the room. *In addition*, get
+		// all `m.space.child` and `m.space.parent` state events which *point to* (via `state_key` or `content.room_id`)
 		// this room. This requires servers to store reverse lookups.
 		refs, err := w.references(roomID)
 		if err != nil {
@@ -196,9 +197,10 @@ func (w *walker) walk() *SpacesResponse {
 		if !w.alreadySent(roomID) {
 			pubRoom := w.publicRoomsChunk(roomID)
 			roomType := ""
-			create := w.stateEvent(roomID, "m.room.create", "")
+			create := w.stateEvent(roomID, gomatrixserverlib.MRoomCreate, "")
 			if create != nil {
-				roomType = gjson.GetBytes(create.Content(), ConstCreateEventContentKey).Str
+				// escape the `.`s so gjson doesn't think it's nested
+				roomType = gjson.GetBytes(create.Content(), strings.ReplaceAll(ConstCreateEventContentKey, ".", `\.`)).Str
 			}
 
 			// Add the total number of events to `PublicRoomsChunk` under `num_refs`. Add `PublicRoomsChunk` to `rooms`.
@@ -333,7 +335,12 @@ func (w *walker) references(roomID string) (eventLookup, error) {
 	}
 	el := make(eventLookup)
 	for _, ev := range events {
-		el.set(ev)
+		// only return events that have a `via` key as per MSC1772
+		// else we'll incorrectly walk redacted events (as the link
+		// is in the state_key)
+		if gjson.GetBytes(ev.Content(), "via").Exists() {
+			el.set(ev)
+		}
 	}
 	return el, nil
 }

setup/mscs/msc2946/msc2946_test.go

@@ -86,8 +86,7 @@ func TestMSC2946(t *testing.T) {
 		Type:     msc2946.ConstSpaceChildEventType,
 		StateKey: &room1,
 		Content: map[string]interface{}{
-			"via":     []string{"localhost"},
-			"present": true,
+			"via": []string{"localhost"},
 		},
 	})
 	rootToR2 := mustCreateEvent(t, fledglingEvent{
@@ -96,8 +95,7 @@ func TestMSC2946(t *testing.T) {
 		Type:     msc2946.ConstSpaceChildEventType,
 		StateKey: &room2,
 		Content: map[string]interface{}{
-			"via":     []string{"localhost"},
-			"present": true,
+			"via": []string{"localhost"},
 		},
 	})
 	rootToS1 := mustCreateEvent(t, fledglingEvent{
@@ -106,8 +104,7 @@ func TestMSC2946(t *testing.T) {
 		Type:     msc2946.ConstSpaceChildEventType,
 		StateKey: &subSpaceS1,
 		Content: map[string]interface{}{
-			"via":     []string{"localhost"},
-			"present": true,
+			"via": []string{"localhost"},
 		},
 	})
 	s1ToR3 := mustCreateEvent(t, fledglingEvent{
@@ -116,8 +113,7 @@ func TestMSC2946(t *testing.T) {
 		Type:     msc2946.ConstSpaceChildEventType,
 		StateKey: &room3,
 		Content: map[string]interface{}{
-			"via":     []string{"localhost"},
-			"present": true,
+			"via": []string{"localhost"},
 		},
 	})
 	s1ToR4 := mustCreateEvent(t, fledglingEvent{
@@ -126,8 +122,7 @@ func TestMSC2946(t *testing.T) {
 		Type:     msc2946.ConstSpaceChildEventType,
 		StateKey: &room4,
 		Content: map[string]interface{}{
-			"via":     []string{"localhost"},
-			"present": true,
+			"via": []string{"localhost"},
 		},
 	})
 	s1ToS2 := mustCreateEvent(t, fledglingEvent{
@@ -136,8 +131,7 @@ func TestMSC2946(t *testing.T) {
 		Type:     msc2946.ConstSpaceChildEventType,
 		StateKey: &subSpaceS2,
 		Content: map[string]interface{}{
-			"via":     []string{"localhost"},
-			"present": true,
+			"via": []string{"localhost"},
 		},
 	})
 	// This is a parent link only
@@ -145,11 +139,9 @@ func TestMSC2946(t *testing.T) {
 		RoomID:   room5,
 		Sender:   alice,
 		Type:     msc2946.ConstSpaceParentEventType,
-		StateKey: &empty,
+		StateKey: &subSpaceS2,
 		Content: map[string]interface{}{
-			"room_id": subSpaceS2,
-			"via":     []string{"localhost"},
-			"present": true,
+			"via": []string{"localhost"},
 		},
 	})
 	// history visibility for R4
@@ -254,7 +246,26 @@ func TestMSC2946(t *testing.T) {
 		if len(res.Rooms) != len(allRooms) {
 			t.Errorf("got %d rooms, want %d", len(res.Rooms), len(allRooms))
 		}
+	})
+	t.Run("can update the graph", func(t *testing.T) {
+		// remove R3 from the graph
+		rmS1ToR3 := mustCreateEvent(t, fledglingEvent{
+			RoomID:   subSpaceS1,
+			Sender:   alice,
+			Type:     msc2946.ConstSpaceChildEventType,
+			StateKey: &room3,
+			Content:  map[string]interface{}{}, // redacted
+		})
+		nopRsAPI.events[rmS1ToR3.EventID()] = rmS1ToR3
+		hooks.Run(hooks.KindNewEventPersisted, rmS1ToR3)
 
+		res := postSpaces(t, 200, "alice", rootSpace, newReq(t, map[string]interface{}{}))
+		if len(res.Events) != 6 { // one less since we don't return redacted events
+			t.Errorf("got %d events, want 6", len(res.Events))
+		}
+		if len(res.Rooms) != (len(allRooms) - 1) { // one less due to lack of R3
+			t.Errorf("got %d rooms, want %d", len(res.Rooms), len(allRooms)-1)
+		}
 	})
 }
 

setup/mscs/msc2946/storage.go

@@ -22,7 +22,6 @@ import (
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/tidwall/gjson"
 )
 
 var (
@@ -81,7 +80,7 @@ func newPostgresDatabase(dbOpts *config.DatabaseOptions) (Database, error) {
 	if d.insertEdgeStmt, err = d.db.Prepare(`
 		INSERT INTO msc2946_edges(room_version, source_room_id, dest_room_id, rel_type, event_json)
 		VALUES($1, $2, $3, $4, $5)
-		ON CONFLICT DO NOTHING
+		ON CONFLICT ON CONSTRAINT msc2946_edges_uniq DO UPDATE SET event_json = $5
 	`); err != nil {
 		return nil, err
 	}
@@ -121,7 +120,7 @@ func newSQLiteDatabase(dbOpts *config.DatabaseOptions) (Database, error) {
 	if d.insertEdgeStmt, err = d.db.Prepare(`
 		INSERT INTO msc2946_edges(room_version, source_room_id, dest_room_id, rel_type, event_json)
 		VALUES($1, $2, $3, $4, $5)
-		ON CONFLICT DO NOTHING
+		ON CONFLICT (source_room_id, dest_room_id, rel_type) DO UPDATE SET event_json = $5
 	`); err != nil {
 		return nil, err
 	}
@@ -175,7 +174,7 @@ func SpaceTarget(he *gomatrixserverlib.HeaderedEvent) string {
 	}
 	switch he.Type() {
 	case ConstSpaceParentEventType:
-		return gjson.GetBytes(he.Content(), "room_id").Str
+		return *he.StateKey()
 	case ConstSpaceChildEventType:
 		return *he.StateKey()
 	}