Merge branch 'master' into kegan/msc-2946-4

.github/workflows/docker-hub.yml

@@ -0,0 +1,71 @@
+# Based on https://github.com/docker/build-push-action
+
+name: "Docker Hub"
+
+on:
+  release:
+    types: [published]
+
+env:
+  DOCKER_NAMESPACE: matrixdotorg
+  DOCKER_HUB_USER: dendritegithub
+  PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7
+
+jobs:
+  Monolith:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Get release tag
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1 
+        with:
+          username: ${{ env.DOCKER_HUB_USER }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Build monolith image
+        id: docker_build_monolith
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./build/docker/Dockerfile.monolith
+          platforms: ${{ env.PLATFORMS }}
+          push: true
+          tags: |
+            ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:latest
+            ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ env.RELEASE_VERSION }}
+
+  Polylith:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Get release tag
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1 
+        with:
+          username: ${{ env.DOCKER_HUB_USER }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Build polylith image
+        id: docker_build_polylith
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./build/docker/Dockerfile.polylith
+          platforms: ${{ env.PLATFORMS }}
+          push: true
+          tags: |
+            ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:latest
+            ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}

CHANGES.md

@@ -1,12 +1,31 @@
 # Changelog
 
+## Dendrite 0.3.6 (2021-01-18)
+
+### Features
+
+* Experimental support for MSC2946 (Spaces Summary) has been merged
+* Send-to-device messages have been refactored and now take advantage of having their own stream position, making delivery more reliable
+* Unstable features and MSCs are now listed in `/versions` (contributed by [sumitks866](https://github.com/sumitks866))
+* Well-known and DNS SRV record results for federated servers are now cached properly, improving outbound federation performance and reducing traffic
+
+### Fixes
+
+* Updating forward extremities will no longer result in so many unnecessary state snapshots, reducing on-going disk usage in the roomserver database
+* Pagination tokens for `/messages` have been fixed, which should improve the reliability of scrollback/pagination
+* Dendrite now avoids returning `null`s in fields of the `/sync` response, and omitting some fields altogether when not needed, which should fix sync issues with Element Android
+* Requests for user device lists now time out quicker, which prevents federated `/send` requests from also timing out in many cases
+* Empty push rules are no longer sent over and over again in `/sync`
+* An integer overflow in the device list updater which could result in panics on 32-bit platforms has been fixed (contributed by [Lesterpig](https://github.com/Lesterpig))
+* Event IDs are now logged properly in federation sender and sync API consumer errors
+
 ## Dendrite 0.3.5 (2021-01-11)
 
 ### Features
 
 * All `/sync` streams are now logically separate after a refactoring exercise
 
-## Fixes
+### Fixes
 
 * Event references are now deeply checked properly when calculating forward extremities, reducing the amount of forward extremities in most cases, which improves RAM utilisation and reduces the work done by state resolution
 * Sync no longer sends incorrect `next_batch` tokens with old stream positions, reducing flashbacks of old messages in clients

build.sh

@@ -17,6 +17,8 @@ else
     export FLAGS=""
 fi
 
-go install -trimpath -ldflags "$FLAGS" -v $PWD/`dirname $0`/cmd/...
+mkdir -p bin
 
-GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs
+CGO_ENABLED=1 go build -trimpath -ldflags "$FLAGS" -v -o "bin/" ./cmd/...
+
+CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs

build/docker/Dockerfile

@@ -1,10 +0,0 @@
-FROM docker.io/golang:1.15-alpine AS builder
-
-RUN apk --update --no-cache add bash build-base
-
-WORKDIR /build
-
-COPY . /build
-
-RUN mkdir -p bin
-RUN sh ./build.sh

build/docker/Dockerfile.monolith

@@ -1,11 +1,20 @@
-FROM matrixdotorg/dendrite:latest AS base
+FROM docker.io/golang:1.15-alpine AS base
+
+RUN apk --update --no-cache add bash build-base
+
+WORKDIR /build
+
+COPY . /build
+
+RUN mkdir -p bin
+RUN go build -trimpath -o bin/ ./cmd/dendrite-monolith-server
+RUN go build -trimpath -o bin/ ./cmd/goose
+RUN go build -trimpath -o bin/ ./cmd/create-account
+RUN go build -trimpath -o bin/ ./cmd/generate-keys
 
 FROM alpine:latest
 
-COPY --from=base /build/bin/dendrite-monolith-server /usr/bin
-COPY --from=base /build/bin/goose /usr/bin
-COPY --from=base /build/bin/create-account /usr/bin
-COPY --from=base /build/bin/generate-keys /usr/bin
+COPY --from=base /build/bin/* /usr/bin
 
 VOLUME /etc/dendrite
 WORKDIR /etc/dendrite

build/docker/Dockerfile.polylith

@@ -1,11 +1,20 @@
-FROM matrixdotorg/dendrite:latest AS base
+FROM docker.io/golang:1.15-alpine AS base
+
+RUN apk --update --no-cache add bash build-base
+
+WORKDIR /build
+
+COPY . /build
+
+RUN mkdir -p bin
+RUN go build -trimpath -o bin/ ./cmd/dendrite-polylith-multi
+RUN go build -trimpath -o bin/ ./cmd/goose
+RUN go build -trimpath -o bin/ ./cmd/create-account
+RUN go build -trimpath -o bin/ ./cmd/generate-keys
 
 FROM alpine:latest
 
-COPY --from=base /build/bin/dendrite-polylith-multi /usr/bin
-COPY --from=base /build/bin/goose /usr/bin
-COPY --from=base /build/bin/create-account /usr/bin
-COPY --from=base /build/bin/generate-keys /usr/bin
+COPY --from=base /build/bin/* /usr/bin
 
 VOLUME /etc/dendrite
 WORKDIR /etc/dendrite

build/docker/images-build.sh

@@ -6,7 +6,5 @@ TAG=${1:-latest}
 
 echo "Building tag '${TAG}'"
 
-docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:${TAG} .
-
 docker build -t matrixdotorg/dendrite-monolith:${TAG}  -f build/docker/Dockerfile.monolith .
 docker build -t matrixdotorg/dendrite-polylith:${TAG}  -f build/docker/Dockerfile.polylith .

clientapi/routing/routing.go

@@ -62,13 +62,19 @@ func Setup(
 	rateLimits := newRateLimits(&cfg.RateLimiting)
 	userInteractiveAuth := auth.NewUserInteractive(accountDB.GetAccountByPassword, cfg)
 
+	unstableFeatures := make(map[string]bool)
+	for _, msc := range cfg.MSCs.MSCs {
+		unstableFeatures["org.matrix."+msc] = true
+	}
+
 	publicAPIMux.Handle("/versions",
 		httputil.MakeExternalAPI("versions", func(req *http.Request) util.JSONResponse {
 			return util.JSONResponse{
 				Code: http.StatusOK,
 				JSON: struct {
-					Versions []string `json:"versions"`
-				}{[]string{
+					Versions         []string        `json:"versions"`
+					UnstableFeatures map[string]bool `json:"unstable_features"`
+				}{Versions: []string{
 					"r0.0.1",
 					"r0.1.0",
 					"r0.2.0",
@@ -76,7 +82,7 @@ func Setup(
 					"r0.4.0",
 					"r0.5.0",
 					"r0.6.1",
-				}},
+				}, UnstableFeatures: unstableFeatures},
 			}
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)

cmd/generate-config/main.go

@@ -63,7 +63,7 @@ func main() {
 	if *defaultsForCI {
 		cfg.ClientAPI.RateLimiting.Enabled = false
 		cfg.FederationSender.DisableTLSValidation = true
-		cfg.MSCs.MSCs = []string{"msc2836"}
+		cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
 		cfg.Logging[0].Level = "trace"
 		// don't hit matrix.org when running tests!!!
 		cfg.SigningKeyServer.KeyPerspectives = config.KeyPerspectives{}

federationsender/consumers/roomserver.go

@@ -102,6 +102,7 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
 			default:
 				// panic rather than continue with an inconsistent database
 				log.WithFields(log.Fields{
+					"event_id":   ev.EventID(),
 					"event":      string(ev.JSON()),
 					"add":        output.NewRoomEvent.AddsStateEventIDs,
 					"del":        output.NewRoomEvent.RemovesStateEventIDs,

internal/version.go

@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 3
-	VersionPatch = 5
+	VersionPatch = 6
 	VersionTag   = "" // example: "rc1"
 )
 

keyserver/internal/device_list_update.go

@@ -245,7 +245,7 @@ func (u *DeviceListUpdater) notifyWorkers(userID string) {
 	}
 	hash := fnv.New32a()
 	_, _ = hash.Write([]byte(remoteServer))
-	index := int(hash.Sum32()) % len(u.workerChans)
+	index := int(int64(hash.Sum32()) % int64(len(u.workerChans)))
 
 	ch := u.assignChannel(userID)
 	u.workerChans[index] <- remoteServer

roomserver/internal/input/input_latest_events.go

@@ -100,7 +100,8 @@ type latestEventsUpdater struct {
 	// The eventID of the event that was processed before this one.
 	lastEventIDSent string
 	// The latest events in the room after processing this event.
-	latest []types.StateAtEventAndReference
+	oldLatest []types.StateAtEventAndReference
+	latest    []types.StateAtEventAndReference
 	// The state entries removed from and added to the current state of the
 	// room as a result of processing this event. They are sorted lists.
 	removed []types.StateEntry
@@ -123,10 +124,10 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error {
 	// state snapshot from somewhere else, e.g. a federated room join,
 	// then start with an empty set - none of the forward extremities
 	// that we knew about before matter anymore.
-	oldLatest := []types.StateAtEventAndReference{}
+	u.oldLatest = []types.StateAtEventAndReference{}
 	if !u.rewritesState {
 		u.oldStateNID = u.updater.CurrentStateSnapshotNID()
-		oldLatest = u.updater.LatestEvents()
+		u.oldLatest = u.updater.LatestEvents()
 	}
 
 	// If the event has already been written to the output log then we
@@ -140,7 +141,7 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error {
 	// Work out what the latest events are. This will include the new
 	// event if it is not already referenced.
 	extremitiesChanged, err := u.calculateLatest(
-		oldLatest, u.event,
+		u.oldLatest, u.event,
 		types.StateAtEventAndReference{
 			EventReference: u.event.EventReference(),
 			StateAtEvent:   u.stateAtEvent,
@@ -200,6 +201,37 @@ func (u *latestEventsUpdater) latestState() error {
 	var err error
 	roomState := state.NewStateResolution(u.api.DB, *u.roomInfo)
 
+	// Work out if the state at the extremities has actually changed
+	// or not. If they haven't then we won't bother doing all of the
+	// hard work.
+	if u.event.StateKey() == nil {
+		stateChanged := false
+		oldStateNIDs := make([]types.StateSnapshotNID, 0, len(u.oldLatest))
+		newStateNIDs := make([]types.StateSnapshotNID, 0, len(u.latest))
+		for _, old := range u.oldLatest {
+			oldStateNIDs = append(oldStateNIDs, old.BeforeStateSnapshotNID)
+		}
+		for _, new := range u.latest {
+			newStateNIDs = append(newStateNIDs, new.BeforeStateSnapshotNID)
+		}
+		oldStateNIDs = state.UniqueStateSnapshotNIDs(oldStateNIDs)
+		newStateNIDs = state.UniqueStateSnapshotNIDs(newStateNIDs)
+		if len(oldStateNIDs) != len(newStateNIDs) {
+			stateChanged = true
+		} else {
+			for i := range oldStateNIDs {
+				if oldStateNIDs[i] != newStateNIDs[i] {
+					stateChanged = true
+					break
+				}
+			}
+		}
+		if !stateChanged {
+			u.newStateNID = u.oldStateNID
+			return nil
+		}
+	}
+
 	// Get a list of the current latest events. This may or may not
 	// include the new event from the input path, depending on whether
 	// it is a forward extremity or not.

roomserver/state/state.go

@@ -116,7 +116,7 @@ func (v StateResolution) LoadCombinedStateAfterEvents(
 	// Deduplicate the IDs before passing them to the database.
 	// There could be duplicates because the events could be state events where
 	// the snapshot of the room state before them was the same.
-	stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, uniqueStateSnapshotNIDs(stateNIDs))
+	stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, UniqueStateSnapshotNIDs(stateNIDs))
 	if err != nil {
 		return nil, fmt.Errorf("v.db.StateBlockNIDs: %w", err)
 	}
@@ -1103,7 +1103,7 @@ func (s stateNIDSorter) Len() int           { return len(s) }
 func (s stateNIDSorter) Less(i, j int) bool { return s[i] < s[j] }
 func (s stateNIDSorter) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
 
-func uniqueStateSnapshotNIDs(nids []types.StateSnapshotNID) []types.StateSnapshotNID {
+func UniqueStateSnapshotNIDs(nids []types.StateSnapshotNID) []types.StateSnapshotNID {
 	return nids[:util.SortAndUnique(stateNIDSorter(nids))]
 }
 

setup/config/config.go

@@ -344,6 +344,7 @@ func (c *Dendrite) Wiring() {
 
 	c.ClientAPI.Derived = &c.Derived
 	c.AppServiceAPI.Derived = &c.Derived
+	c.ClientAPI.MSCs = &c.MSCs
 }
 
 // Error returns a string detailing how many errors were contained within a

setup/config/config_clientapi.go

@@ -37,6 +37,8 @@ type ClientAPI struct {
 
 	// Rate-limiting options
 	RateLimiting RateLimiting `yaml:"rate_limiting"`
+
+	MSCs *MSCs `yaml:"mscs"`
 }
 
 func (c *ClientAPI) Defaults() {

setup/mscs/msc2946/msc2946.go

@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"strings"
 	"sync"
 	"time"
 
@@ -250,7 +251,8 @@ func (w *walker) walk() *gomatrixserverlib.MSC2946SpacesResponse {
 			roomType := ""
 			create := w.stateEvent(roomID, gomatrixserverlib.MRoomCreate, "")
 			if create != nil {
-				roomType = gjson.GetBytes(create.Content(), ConstCreateEventContentKey).Str
+				// escape the `.`s so gjson doesn't think it's nested
+				roomType = gjson.GetBytes(create.Content(), strings.ReplaceAll(ConstCreateEventContentKey, ".", `\.`)).Str
 			}
 
 			// Add the total number of events to `PublicRoomsChunk` under `num_refs`. Add `PublicRoomsChunk` to `rooms`.

syncapi/consumers/roomserver.go

@@ -173,6 +173,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent(
 	if err != nil {
 		// panic rather than continue with an inconsistent database
 		log.WithFields(log.Fields{
+			"event_id":   ev.EventID(),
 			"event":      string(ev.JSON()),
 			log.ErrorKey: err,
 			"add":        msg.AddsStateEventIDs,
@@ -215,6 +216,7 @@ func (s *OutputRoomEventConsumer) onOldRoomEvent(
 	if err != nil {
 		// panic rather than continue with an inconsistent database
 		log.WithFields(log.Fields{
+			"event_id":   ev.EventID(),
 			"event":      string(ev.JSON()),
 			log.ErrorKey: err,
 		}).Panicf("roomserver output log: write old event failure")
@@ -276,6 +278,7 @@ func (s *OutputRoomEventConsumer) onNewInviteEvent(
 	if err != nil {
 		// panic rather than continue with an inconsistent database
 		log.WithFields(log.Fields{
+			"event_id":   msg.Event.EventID(),
 			"event":      string(msg.Event.JSON()),
 			"pdupos":     pduPos,
 			log.ErrorKey: err,