Merge branch 'master' into kegan/syncapi-table-even-moar

cmd/dendrite-key-server/main.go

@@ -0,0 +1,34 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"github.com/matrix-org/dendrite/common/basecomponent"
+	"github.com/matrix-org/dendrite/keyserver"
+)
+
+func main() {
+	cfg := basecomponent.ParseFlags()
+	base := basecomponent.NewBaseDendrite(cfg, "KeyServer")
+	defer base.Close() // nolint: errcheck
+
+	accountDB := base.CreateAccountsDB()
+	deviceDB := base.CreateDeviceDB()
+
+	keyserver.SetupKeyServerComponent(base, deviceDB, accountDB)
+
+	base.SetupAndServeHTTP(string(base.Cfg.Bind.KeyServer), string(base.Cfg.Listen.KeyServer))
+
+}

cmd/dendrite-monolith-server/main.go

@@ -29,6 +29,7 @@ import (
 	"github.com/matrix-org/dendrite/eduserver/cache"
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/federationsender"
+	"github.com/matrix-org/dendrite/keyserver"
 	"github.com/matrix-org/dendrite/mediaapi"
 	"github.com/matrix-org/dendrite/publicroomsapi"
 	"github.com/matrix-org/dendrite/publicroomsapi/storage"
@@ -76,6 +77,9 @@ func main() {
 		federation, &keyRing, rsAPI,
 		eduInputAPI, asAPI, transactions.New(), fsAPI,
 	)
+	keyserver.SetupKeyServerComponent(
+		base, deviceDB, accountDB,
+	)
 	eduProducer := producers.NewEDUServerProducer(eduInputAPI)
 	federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asAPI, fsAPI, eduProducer)
 	mediaapi.SetupMediaAPIComponent(base, deviceDB)

common/config/config.go

@@ -229,6 +229,7 @@ type Dendrite struct {
 		FederationSender Address `yaml:"federation_sender"`
 		PublicRoomsAPI   Address `yaml:"public_rooms_api"`
 		EDUServer        Address `yaml:"edu_server"`
+		KeyServer        Address `yaml:"key_server"`
 	} `yaml:"bind"`
 
 	// The addresses for talking to other microservices.
@@ -242,6 +243,7 @@ type Dendrite struct {
 		FederationSender Address `yaml:"federation_sender"`
 		PublicRoomsAPI   Address `yaml:"public_rooms_api"`
 		EDUServer        Address `yaml:"edu_server"`
+		KeyServer        Address `yaml:"key_server"`
 	} `yaml:"listen"`
 
 	// The config for tracing the dendrite servers.

dendrite-config.yaml

@@ -137,6 +137,7 @@ listen:
     federation_sender: "localhost:7776"
     appservice_api: "localhost:7777"
     edu_server: "localhost:7778"
+    key_server: "localhost:7779"
 
 # The configuration for tracing the dendrite components.
 tracing:

docker/docker-compose.yml

@@ -154,6 +154,16 @@ services:
     networks:
       - internal
       
+  key_server:
+    container_name: dendrite_key_server
+    hostname: key_server
+    entrypoint: ["bash", "./docker/services/key-server.sh"]
+    build: ./
+    volumes:
+        - ..:/build
+    networks:
+        - internal
+
   postgres:
     container_name: dendrite_postgres
     hostname: postgres

docker/services/key-server.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+bash ./docker/build.sh
+
+./bin/dendrite-key-server --config dendrite.yaml

keyserver/keyserver.go

@@ -0,0 +1,32 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package keyserver
+
+import (
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
+	"github.com/matrix-org/dendrite/common/basecomponent"
+	"github.com/matrix-org/dendrite/keyserver/routing"
+)
+
+// SetupFederationSenderComponent sets up and registers HTTP handlers for the
+// FederationSender component.
+func SetupKeyServerComponent(
+	base *basecomponent.BaseDendrite,
+	deviceDB devices.Database,
+	accountsDB accounts.Database,
+) {
+	routing.Setup(base.APIMux, base.Cfg, accountsDB, deviceDB)
+}

keyserver/routing/keys.go

@@ -0,0 +1,33 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+
+	"github.com/matrix-org/util"
+)
+
+func QueryKeys(
+	req *http.Request,
+) util.JSONResponse {
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: map[string]interface{}{
+			"failures":    map[string]interface{}{},
+			"device_keys": map[string]interface{}{},
+		},
+	}
+}

keyserver/routing/routing.go

@@ -0,0 +1,56 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/clientapi/auth"
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/util"
+)
+
+const pathPrefixR0 = "/_matrix/client/r0"
+
+// Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client
+// to clients which need to make outbound HTTP requests.
+//
+// Due to Setup being used to call many other functions, a gocyclo nolint is
+// applied:
+// nolint: gocyclo
+func Setup(
+	apiMux *mux.Router, cfg *config.Dendrite,
+	accountDB accounts.Database,
+	deviceDB devices.Database,
+) {
+	r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter()
+
+	authData := auth.Data{
+		AccountDB:   accountDB,
+		DeviceDB:    deviceDB,
+		AppServices: cfg.Derived.ApplicationServices,
+	}
+
+	r0mux.Handle("/keys/query",
+		common.MakeAuthAPI("queryKeys", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			return QueryKeys(req)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+}

roomserver/internal/input_membership.go

@@ -231,8 +231,7 @@ func updateToLeaveMembership(
 	return updates, nil
 }
 
-// membershipChanges pairs up the membership state changes from a sorted list
-// of state removed and a sorted list of state added.
+// membershipChanges pairs up the membership state changes.
 func membershipChanges(removed, added []types.StateEntry) []stateChange {
 	changes := pairUpChanges(removed, added)
 	var result []stateChange
@@ -251,64 +250,39 @@ type stateChange struct {
 }
 
 // pairUpChanges pairs up the state events added and removed for each type,
-// state key tuple. Assumes that removed and added are sorted.
+// state key tuple.
 func pairUpChanges(removed, added []types.StateEntry) []stateChange {
-	var ai int
-	var ri int
-	var result []stateChange
-	for {
-		switch {
-		case ai == len(added):
-			// We've reached the end of the added entries.
-			// The rest of the removed list are events that were removed without
-			// an event with the same state key being added.
-			for _, s := range removed[ri:] {
-				result = append(result, stateChange{
-					StateKeyTuple:   s.StateKeyTuple,
-					removedEventNID: s.EventNID,
-				})
-			}
-			return result
-		case ri == len(removed):
-			// We've reached the end of the removed entries.
-			// The rest of the added list are events that were added without
-			// an event with the same state key being removed.
-			for _, s := range added[ai:] {
-				result = append(result, stateChange{
-					StateKeyTuple: s.StateKeyTuple,
-					addedEventNID: s.EventNID,
-				})
-			}
-			return result
-		case added[ai].StateKeyTuple == removed[ri].StateKeyTuple:
-			// The tuple is in both lists so an event with that key is being
-			// removed and another event with the same key is being added.
-			result = append(result, stateChange{
-				StateKeyTuple:   added[ai].StateKeyTuple,
-				removedEventNID: removed[ri].EventNID,
-				addedEventNID:   added[ai].EventNID,
-			})
-			ai++
-			ri++
-		case added[ai].StateKeyTuple.LessThan(removed[ri].StateKeyTuple):
-			// The lists are sorted so the added entry being less than the
-			// removed entry means that the added event was added without an
-			// event with the same key being removed.
-			result = append(result, stateChange{
-				StateKeyTuple: added[ai].StateKeyTuple,
-				addedEventNID: added[ai].EventNID,
-			})
-			ai++
-		default:
-			// Reaching the default case implies that the removed entry is less
-			// than the added entry. Since the lists are sorted this means that
-			// the removed event was removed without an event with the same
-			// key being added.
-			result = append(result, stateChange{
-				StateKeyTuple:   removed[ai].StateKeyTuple,
-				removedEventNID: removed[ri].EventNID,
-			})
-			ri++
+	tuples := make(map[types.StateKeyTuple]stateChange)
+	changes := []stateChange{}
+
+	// First, go through the newly added state entries.
+	for _, add := range added {
+		if change, ok := tuples[add.StateKeyTuple]; ok {
+			// If we already have an entry, update it.
+			change.addedEventNID = add.EventNID
+			tuples[add.StateKeyTuple] = change
+		} else {
+			// Otherwise, create a new entry.
+			tuples[add.StateKeyTuple] = stateChange{add.StateKeyTuple, 0, add.EventNID}
 		}
 	}
+
+	// Now go through the removed state entries.
+	for _, remove := range removed {
+		if change, ok := tuples[remove.StateKeyTuple]; ok {
+			// If we already have an entry, update it.
+			change.removedEventNID = remove.EventNID
+			tuples[remove.StateKeyTuple] = change
+		} else {
+			// Otherwise, create a new entry.
+			tuples[remove.StateKeyTuple] = stateChange{remove.StateKeyTuple, remove.EventNID, 0}
+		}
+	}
+
+	// Now return the changes as an array.
+	for _, change := range tuples {
+		changes = append(changes, change)
+	}
+
+	return changes
 }

roomserver/internal/perform_join.go

@@ -121,6 +121,22 @@ func (r *RoomserverInternalAPI) performJoinRoomByID(
 		return fmt.Errorf("eb.SetContent: %w", err)
 	}
 
+	// First work out if this is in response to an existing invite.
+	// If it is then we avoid the situation where we might think we
+	// know about a room in the following section but don't know the
+	// latest state as all of our users have left.
+	isInvitePending, inviteSender, err := r.isInvitePending(ctx, req.RoomIDOrAlias, req.UserID)
+	if err == nil && isInvitePending {
+		// Add the server of the person who invited us to the server list,
+		// as they should be a fairly good bet.
+		if _, inviterDomain, ierr := gomatrixserverlib.SplitID('@', inviteSender); ierr == nil {
+			req.ServerNames = append(req.ServerNames, inviterDomain)
+		}
+
+		// Perform a federated room join.
+		return r.performFederatedJoinRoomByID(ctx, req, res)
+	}
+
 	// Try to construct an actual join event from the template.
 	// If this succeeds then it is a sign that the room already exists
 	// locally on the homeserver.
@@ -178,6 +194,22 @@ func (r *RoomserverInternalAPI) performJoinRoomByID(
 			return fmt.Errorf("Room ID %q does not exist", req.RoomIDOrAlias)
 		}
 
+		// Perform a federated room join.
+		return r.performFederatedJoinRoomByID(ctx, req, res)
+
+	default:
+		// Something else went wrong.
+		return fmt.Errorf("Error joining local room: %q", err)
+	}
+
+	return nil
+}
+
+func (r *RoomserverInternalAPI) performFederatedJoinRoomByID(
+	ctx context.Context,
+	req *api.PerformJoinRequest,
+	res *api.PerformJoinResponse, // nolint:unparam
+) error {
 	// Try joining by all of the supplied server names.
 	fedReq := fsAPI.PerformJoinRequest{
 		RoomID:      req.RoomIDOrAlias, // the room ID to try and join
@@ -186,14 +218,9 @@ func (r *RoomserverInternalAPI) performJoinRoomByID(
 		Content:     req.Content,       // the membership event content
 	}
 	fedRes := fsAPI.PerformJoinResponse{}
-		err = r.fsAPI.PerformJoin(ctx, &fedReq, &fedRes)
-		if err != nil {
+	if err := r.fsAPI.PerformJoin(ctx, &fedReq, &fedRes); err != nil {
 		return fmt.Errorf("Error joining federated room: %q", err)
 	}
 
-	default:
-		return fmt.Errorf("Error joining room %q: %w", req.RoomIDOrAlias, err)
-	}
-
 	return nil
 }

roomserver/internal/perform_leave.go

@@ -38,7 +38,7 @@ func (r *RoomserverInternalAPI) performLeaveRoomByID(
 ) error {
 	// If there's an invite outstanding for the room then respond to
 	// that.
-	isInvitePending, senderUser, err := r.isInvitePending(ctx, req, res)
+	isInvitePending, senderUser, err := r.isInvitePending(ctx, req.RoomID, req.UserID)
 	if err == nil && isInvitePending {
 		return r.performRejectInvite(ctx, req, res, senderUser)
 	}
@@ -160,23 +160,22 @@ func (r *RoomserverInternalAPI) performRejectInvite(
 
 func (r *RoomserverInternalAPI) isInvitePending(
 	ctx context.Context,
-	req *api.PerformLeaveRequest,
-	res *api.PerformLeaveResponse, // nolint:unparam
+	roomID, userID string,
 ) (bool, string, error) {
 	// Look up the room NID for the supplied room ID.
-	roomNID, err := r.DB.RoomNID(ctx, req.RoomID)
+	roomNID, err := r.DB.RoomNID(ctx, roomID)
 	if err != nil {
 		return false, "", fmt.Errorf("r.DB.RoomNID: %w", err)
 	}
 
 	// Look up the state key NID for the supplied user ID.
-	targetUserNIDs, err := r.DB.EventStateKeyNIDs(ctx, []string{req.UserID})
+	targetUserNIDs, err := r.DB.EventStateKeyNIDs(ctx, []string{userID})
 	if err != nil {
 		return false, "", fmt.Errorf("r.DB.EventStateKeyNIDs: %w", err)
 	}
-	targetUserNID, targetUserFound := targetUserNIDs[req.UserID]
+	targetUserNID, targetUserFound := targetUserNIDs[userID]
 	if !targetUserFound {
-		return false, "", fmt.Errorf("missing NID for user %q (%+v)", req.UserID, targetUserNIDs)
+		return false, "", fmt.Errorf("missing NID for user %q (%+v)", userID, targetUserNIDs)
 	}
 
 	// Let's see if we have an event active for the user in the room. If