dendrite/federationapi/routing/routing.go
Matthew Hodgson 0571d395b5
Peeking over federation via MSC2444 (#1391)
* a very very WIP first cut of peeking via MSC2753.

doesn't yet compile or work.
needs to actually add the peeking block into the sync response.
checking in now before it gets any bigger, and to gather any initial feedback on the vague shape of it.

* make PeekingDeviceSet private

* add server_name param

* blind stab at adding a `peek` section to /sync

* make it build

* make it launch

* add peeking to getResponseWithPDUsForCompleteSync

* cancel any peeks when we join a room

* spell out how to runoutside of docker if you want speed

* fix SQL

* remove unnecessary txn for SelectPeeks

* fix s/join/peek/ cargocult fail

* HACK: Track goroutine IDs to determine when we write by the wrong thread

To use: set `DENDRITE_TRACE_SQL=1` then grep for `unsafe`

* Track partition offsets and only log unsafe for non-selects

* Put redactions in the writer goroutine

* Update filters on writer goroutine

* wrap peek storage in goid hack

* use exclusive writer, and MarkPeeksAsOld more efficiently

* don't log ascii in binary at sql trace...

* strip out empty roomd deltas

* re-add txn to SelectPeeks

* re-add accidentally deleted field

* reject peeks for non-worldreadable rooms

* move perform_peek

* fix package

* correctly refactor perform_peek

* WIP of implementing MSC2444

* typo

* Revert "Merge branch 'kegan/HACK-goid-sqlite-db-is-locked' into matthew/peeking"

This reverts commit 3cebd8dbfb, reversing
changes made to ed4b3a58a7.

* (almost) make it build

* clean up bad merge

* support SendEventWithState with optional event

* fix build & lint

* fix build & lint

* reinstate federated peeks in the roomserver (doh)

* fix sql thinko

* todo for authenticating state returned by /peek

* support returning current state from QueryStateAndAuthChain

* handle SS /peek

* reimplement SS /peek to prod the RS to tell the FS about the peek

* rename RemotePeeks as OutboundPeeks

* rename remote_peeks_table as outbound_peeks_table

* add perform_handle_remote_peek.go

* flesh out federation doc

* add inbound peeks table and hook it up

* rename ambiguous RemotePeek as InboundPeek

* rename FSAPI's PerformPeek as PerformOutboundPeek

* setup inbound peeks db correctly

* fix api.SendEventWithState with no event

* track latestevent on /peek

* go fmt

* document the peek send stream race better

* fix SendEventWithRewrite not to bail if handed a non-state event

* add fixme

* switch SS /peek to use SendEventWithRewrite

* fix comment

* use reverse topo ordering to find latest extrem

* support postgres for federated peeking

* go fmt

* back out bogus go.mod change

* Fix performOutboundPeekUsingServer

* Fix getAuthChain -> GetAuthChain

* Fix build issues

* Fix build again

* Fix getAuthChain -> GetAuthChain

* Don't repeat outbound peeks for the same room ID to the same servers

* Fix lint

* Don't omitempty to appease sytest

Co-authored-by: Kegan Dougal <kegan@matrix.org>
Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
2021-01-22 14:55:08 +00:00

461 lines
18 KiB
Go

// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package routing
import (
"net/http"
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
"github.com/matrix-org/dendrite/internal/httputil"
keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
// Setup registers HTTP handlers with the given ServeMux.
// The provided publicAPIMux MUST have `UseEncodedPath()` enabled or else routes will incorrectly
// path unescape twice (once from the router, once from MakeFedAPI). We need to have this enabled
// so we can decode paths like foo/bar%2Fbaz as [foo, bar/baz] - by default it will decode to [foo, bar, baz]
//
// Due to Setup being used to call many other functions, a gocyclo nolint is
// applied:
// nolint: gocyclo
func Setup(
fedMux, keyMux *mux.Router,
cfg *config.FederationAPI,
rsAPI roomserverAPI.RoomserverInternalAPI,
eduAPI eduserverAPI.EDUServerInputAPI,
fsAPI federationSenderAPI.FederationSenderInternalAPI,
keys gomatrixserverlib.JSONVerifier,
federation *gomatrixserverlib.FederationClient,
userAPI userapi.UserInternalAPI,
keyAPI keyserverAPI.KeyInternalAPI,
) {
v2keysmux := keyMux.PathPrefix("/v2").Subrouter()
v1fedmux := fedMux.PathPrefix("/v1").Subrouter()
v2fedmux := fedMux.PathPrefix("/v2").Subrouter()
wakeup := &httputil.FederationWakeups{
FsAPI: fsAPI,
}
localKeys := httputil.MakeExternalAPI("localkeys", func(req *http.Request) util.JSONResponse {
return LocalKeys(cfg)
})
notaryKeys := httputil.MakeExternalAPI("notarykeys", func(req *http.Request) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
var pkReq *gomatrixserverlib.PublicKeyNotaryLookupRequest
serverName := gomatrixserverlib.ServerName(vars["serverName"])
keyID := gomatrixserverlib.KeyID(vars["keyID"])
if serverName != "" && keyID != "" {
pkReq = &gomatrixserverlib.PublicKeyNotaryLookupRequest{
ServerKeys: map[gomatrixserverlib.ServerName]map[gomatrixserverlib.KeyID]gomatrixserverlib.PublicKeyNotaryQueryCriteria{
serverName: {
keyID: gomatrixserverlib.PublicKeyNotaryQueryCriteria{},
},
},
}
}
return NotaryKeys(req, cfg, fsAPI, pkReq)
})
// Ignore the {keyID} argument as we only have a single server key so we always
// return that key.
// Even if we had more than one server key, we would probably still ignore the
// {keyID} argument and always return a response containing all of the keys.
v2keysmux.Handle("/server/{keyID}", localKeys).Methods(http.MethodGet)
v2keysmux.Handle("/server/", localKeys).Methods(http.MethodGet)
v2keysmux.Handle("/server", localKeys).Methods(http.MethodGet)
v2keysmux.Handle("/query", notaryKeys).Methods(http.MethodPost)
v2keysmux.Handle("/query/{serverName}/{keyID}", notaryKeys).Methods(http.MethodGet)
v1fedmux.Handle("/send/{txnID}", httputil.MakeFedAPI(
"federation_send", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
return Send(
httpReq, request, gomatrixserverlib.TransactionID(vars["txnID"]),
cfg, rsAPI, eduAPI, keyAPI, keys, federation,
)
},
)).Methods(http.MethodPut, http.MethodOptions)
v1fedmux.Handle("/invite/{roomID}/{eventID}", httputil.MakeFedAPI(
"federation_invite", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("Forbidden by server ACLs"),
}
}
return InviteV1(
httpReq, request, vars["roomID"], vars["eventID"],
cfg, rsAPI, keys,
)
},
)).Methods(http.MethodPut, http.MethodOptions)
v2fedmux.Handle("/invite/{roomID}/{eventID}", httputil.MakeFedAPI(
"federation_invite", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("Forbidden by server ACLs"),
}
}
return InviteV2(
httpReq, request, vars["roomID"], vars["eventID"],
cfg, rsAPI, keys,
)
},
)).Methods(http.MethodPut, http.MethodOptions)
v1fedmux.Handle("/3pid/onbind", httputil.MakeExternalAPI("3pid_onbind",
func(req *http.Request) util.JSONResponse {
return CreateInvitesFrom3PIDInvites(req, rsAPI, cfg, federation, userAPI)
},
)).Methods(http.MethodPost, http.MethodOptions)
v1fedmux.Handle("/exchange_third_party_invite/{roomID}", httputil.MakeFedAPI(
"exchange_third_party_invite", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
return ExchangeThirdPartyInvite(
httpReq, request, vars["roomID"], rsAPI, cfg, federation,
)
},
)).Methods(http.MethodPut, http.MethodOptions)
v1fedmux.Handle("/event/{eventID}", httputil.MakeFedAPI(
"federation_get_event", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
return GetEvent(
httpReq.Context(), request, rsAPI, vars["eventID"], cfg.Matrix.ServerName,
)
},
)).Methods(http.MethodGet)
v1fedmux.Handle("/state/{roomID}", httputil.MakeFedAPI(
"federation_get_state", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("Forbidden by server ACLs"),
}
}
return GetState(
httpReq.Context(), request, rsAPI, vars["roomID"],
)
},
)).Methods(http.MethodGet)
v1fedmux.Handle("/state_ids/{roomID}", httputil.MakeFedAPI(
"federation_get_state_ids", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("Forbidden by server ACLs"),
}
}
return GetStateIDs(
httpReq.Context(), request, rsAPI, vars["roomID"],
)
},
)).Methods(http.MethodGet)
v1fedmux.Handle("/event_auth/{roomID}/{eventID}", httputil.MakeFedAPI(
"federation_get_event_auth", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("Forbidden by server ACLs"),
}
}
return GetEventAuth(
httpReq.Context(), request, rsAPI, vars["roomID"], vars["eventID"],
)
},
)).Methods(http.MethodGet)
v1fedmux.Handle("/query/directory", httputil.MakeFedAPI(
"federation_query_room_alias", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
return RoomAliasToID(
httpReq, federation, cfg, rsAPI, fsAPI,
)
},
)).Methods(http.MethodGet)
v1fedmux.Handle("/query/profile", httputil.MakeFedAPI(
"federation_query_profile", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
return GetProfile(
httpReq, userAPI, cfg,
)
},
)).Methods(http.MethodGet)
v1fedmux.Handle("/user/devices/{userID}", httputil.MakeFedAPI(
"federation_user_devices", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
return GetUserDevices(
httpReq, keyAPI, vars["userID"],
)
},
)).Methods(http.MethodGet)
v1fedmux.Handle("/peek/{roomID}/{peekID}", httputil.MakeFedAPI(
"federation_peek", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("Forbidden by server ACLs"),
}
}
roomID := vars["roomID"]
peekID := vars["peekID"]
queryVars := httpReq.URL.Query()
remoteVersions := []gomatrixserverlib.RoomVersion{}
if vers, ok := queryVars["ver"]; ok {
// The remote side supplied a ?ver= so use that to build up the list
// of supported room versions
for _, v := range vers {
remoteVersions = append(remoteVersions, gomatrixserverlib.RoomVersion(v))
}
} else {
// The remote side didn't supply a ?ver= so just assume that they only
// support room version 1
remoteVersions = append(remoteVersions, gomatrixserverlib.RoomVersionV1)
}
return Peek(
httpReq, request, cfg, rsAPI, roomID, peekID, remoteVersions,
)
},
)).Methods(http.MethodPut, http.MethodDelete)
v1fedmux.Handle("/make_join/{roomID}/{userID}", httputil.MakeFedAPI(
"federation_make_join", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("Forbidden by server ACLs"),
}
}
roomID := vars["roomID"]
userID := vars["userID"]
queryVars := httpReq.URL.Query()
remoteVersions := []gomatrixserverlib.RoomVersion{}
if vers, ok := queryVars["ver"]; ok {
// The remote side supplied a ?ver= so use that to build up the list
// of supported room versions
for _, v := range vers {
remoteVersions = append(remoteVersions, gomatrixserverlib.RoomVersion(v))
}
} else {
// The remote side didn't supply a ?ver= so just assume that they only
// support room version 1, as per the spec
// https://matrix.org/docs/spec/server_server/r0.1.3#get-matrix-federation-v1-make-join-roomid-userid
remoteVersions = append(remoteVersions, gomatrixserverlib.RoomVersionV1)
}
return MakeJoin(
httpReq, request, cfg, rsAPI, roomID, userID, remoteVersions,
)
},
)).Methods(http.MethodGet)
v1fedmux.Handle("/send_join/{roomID}/{eventID}", httputil.MakeFedAPI(
"federation_send_join", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("Forbidden by server ACLs"),
}
}
roomID := vars["roomID"]
eventID := vars["eventID"]
res := SendJoin(
httpReq, request, cfg, rsAPI, keys, roomID, eventID,
)
// not all responses get wrapped in [code, body]
var body interface{}
body = []interface{}{
res.Code, res.JSON,
}
jerr, ok := res.JSON.(*jsonerror.MatrixError)
if ok {
body = jerr
}
return util.JSONResponse{
Headers: res.Headers,
Code: res.Code,
JSON: body,
}
},
)).Methods(http.MethodPut)
v2fedmux.Handle("/send_join/{roomID}/{eventID}", httputil.MakeFedAPI(
"federation_send_join", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("Forbidden by server ACLs"),
}
}
roomID := vars["roomID"]
eventID := vars["eventID"]
return SendJoin(
httpReq, request, cfg, rsAPI, keys, roomID, eventID,
)
},
)).Methods(http.MethodPut)
v1fedmux.Handle("/make_leave/{roomID}/{eventID}", httputil.MakeFedAPI(
"federation_make_leave", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("Forbidden by server ACLs"),
}
}
roomID := vars["roomID"]
eventID := vars["eventID"]
return MakeLeave(
httpReq, request, cfg, rsAPI, roomID, eventID,
)
},
)).Methods(http.MethodGet)
v1fedmux.Handle("/send_leave/{roomID}/{eventID}", httputil.MakeFedAPI(
"federation_send_leave", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("Forbidden by server ACLs"),
}
}
roomID := vars["roomID"]
eventID := vars["eventID"]
res := SendLeave(
httpReq, request, cfg, rsAPI, keys, roomID, eventID,
)
// not all responses get wrapped in [code, body]
var body interface{}
body = []interface{}{
res.Code, res.JSON,
}
jerr, ok := res.JSON.(*jsonerror.MatrixError)
if ok {
body = jerr
}
return util.JSONResponse{
Headers: res.Headers,
Code: res.Code,
JSON: body,
}
},
)).Methods(http.MethodPut)
v2fedmux.Handle("/send_leave/{roomID}/{eventID}", httputil.MakeFedAPI(
"federation_send_leave", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("Forbidden by server ACLs"),
}
}
roomID := vars["roomID"]
eventID := vars["eventID"]
return SendLeave(
httpReq, request, cfg, rsAPI, keys, roomID, eventID,
)
},
)).Methods(http.MethodPut)
v1fedmux.Handle("/version", httputil.MakeExternalAPI(
"federation_version",
func(httpReq *http.Request) util.JSONResponse {
return Version()
},
)).Methods(http.MethodGet)
v1fedmux.Handle("/get_missing_events/{roomID}", httputil.MakeFedAPI(
"federation_get_missing_events", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("Forbidden by server ACLs"),
}
}
return GetMissingEvents(httpReq, request, rsAPI, vars["roomID"])
},
)).Methods(http.MethodPost)
v1fedmux.Handle("/backfill/{roomID}", httputil.MakeFedAPI(
"federation_backfill", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("Forbidden by server ACLs"),
}
}
return Backfill(httpReq, request, rsAPI, vars["roomID"], cfg)
},
)).Methods(http.MethodGet)
v1fedmux.Handle("/publicRooms",
httputil.MakeExternalAPI("federation_public_rooms", func(req *http.Request) util.JSONResponse {
return GetPostPublicRooms(req, rsAPI)
}),
).Methods(http.MethodGet)
v1fedmux.Handle("/user/keys/claim", httputil.MakeFedAPI(
"federation_keys_claim", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
return ClaimOneTimeKeys(httpReq, request, keyAPI, cfg.Matrix.ServerName)
},
)).Methods(http.MethodPost)
v1fedmux.Handle("/user/keys/query", httputil.MakeFedAPI(
"federation_keys_query", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
return QueryDeviceKeys(httpReq, request, keyAPI, cfg.Matrix.ServerName)
},
)).Methods(http.MethodPost)
}