From 396219ef534093b45ad02cccc8ca9cf0f9742c40 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Mon, 13 Jul 2020 16:02:35 +0100 Subject: [PATCH] Add boilerplate for key server APIs (#1196) Also add a README which outilnes how things will work. --- {keyserver => clientapi}/routing/keys.go | 7 ++ clientapi/routing/routing.go | 13 +++ cmd/dendrite-key-server/main.go | 4 +- cmd/dendrite-monolith-server/main.go | 3 + internal/config/config.go | 11 ++- internal/setup/base.go | 11 +++ internal/setup/monolith.go | 5 +- keyserver/README.md | 19 +++++ keyserver/api/api.go | 49 +++++++++++ keyserver/internal/internal.go | 19 +++++ keyserver/inthttp/client.go | 103 +++++++++++++++++++++++ keyserver/inthttp/server.go | 61 ++++++++++++++ keyserver/keyserver.go | 21 +++-- keyserver/routing/routing.go | 54 ------------ 14 files changed, 312 insertions(+), 68 deletions(-) rename {keyserver => clientapi}/routing/keys.go (88%) create mode 100644 keyserver/README.md create mode 100644 keyserver/api/api.go create mode 100644 keyserver/internal/internal.go create mode 100644 keyserver/inthttp/client.go create mode 100644 keyserver/inthttp/server.go delete mode 100644 keyserver/routing/routing.go diff --git a/keyserver/routing/keys.go b/clientapi/routing/keys.go similarity index 88% rename from keyserver/routing/keys.go rename to clientapi/routing/keys.go index a279a747c..5c1a657fe 100644 --- a/keyserver/routing/keys.go +++ b/clientapi/routing/keys.go @@ -31,3 +31,10 @@ func QueryKeys( }, } } + +func UploadKeys(req *http.Request) util.JSONResponse { + return util.JSONResponse{ + Code: 200, + JSON: struct{}{}, + } +} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index f764bd4d3..965a46d29 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -695,4 +695,17 @@ func Setup( return GetCapabilities(req, rsAPI) }), ).Methods(http.MethodGet) + + r0mux.Handle("/keys/query", + httputil.MakeAuthAPI("queryKeys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + return QueryKeys(req) + }), + ).Methods(http.MethodPost, http.MethodOptions) + + // Supplying a device ID is deprecated. + r0mux.Handle("/keys/upload/{deviceID}", + httputil.MakeAuthAPI("keys_upload", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + return UploadKeys(req) + }), + ).Methods(http.MethodPost, http.MethodOptions) } diff --git a/cmd/dendrite-key-server/main.go b/cmd/dendrite-key-server/main.go index b557cbd9e..813ddddc5 100644 --- a/cmd/dendrite-key-server/main.go +++ b/cmd/dendrite-key-server/main.go @@ -24,9 +24,9 @@ func main() { base := setup.NewBaseDendrite(cfg, "KeyServer", true) defer base.Close() // nolint: errcheck - userAPI := base.UserAPIClient() + intAPI := keyserver.NewInternalAPI() - keyserver.AddPublicRoutes(base.PublicAPIMux, base.Cfg, userAPI) + keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI) base.SetupAndServeHTTP(string(base.Cfg.Bind.KeyServer), string(base.Cfg.Listen.KeyServer)) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 9ac6941b2..83c49d1e1 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -27,6 +27,7 @@ import ( "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/setup" + "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/serverkeyapi" @@ -118,6 +119,7 @@ func main() { rsImpl.SetFederationSenderAPI(fsAPI) stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer) + keyAPI := keyserver.NewInternalAPI() monolith := setup.Monolith{ Config: base.Cfg, @@ -136,6 +138,7 @@ func main() { ServerKeyAPI: serverKeyAPI, StateAPI: stateAPI, UserAPI: userAPI, + KeyAPI: keyAPI, } monolith.AddAllPublicRoutes(base.PublicAPIMux) diff --git a/internal/config/config.go b/internal/config/config.go index 777bd6a39..ac6249d6a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -764,7 +764,7 @@ func (config *Dendrite) FederationSenderURL() string { return "http://" + string(config.Listen.FederationSender) } -// ServerKeyAPIURL returns an HTTP URL for where the federation sender is listening. +// ServerKeyAPIURL returns an HTTP URL for where the server key API is listening. func (config *Dendrite) ServerKeyAPIURL() string { // Hard code the server key API server to talk HTTP for now. // If we support HTTPS we need to think of a practical way to do certificate validation. @@ -773,6 +773,15 @@ func (config *Dendrite) ServerKeyAPIURL() string { return "http://" + string(config.Listen.ServerKeyAPI) } +// KeyServerURL returns an HTTP URL for where the key server is listening. +func (config *Dendrite) KeyServerURL() string { + // Hard code the key server to talk HTTP for now. + // If we support HTTPS we need to think of a practical way to do certificate validation. + // People setting up servers shouldn't need to get a certificate valid for the public + // internet for an internal API. + return "http://" + string(config.Listen.KeyServer) +} + // SetupTracing configures the opentracing using the supplied configuration. func (config *Dendrite) SetupTracing(serviceName string) (closer io.Closer, err error) { if !config.Tracing.Enabled { diff --git a/internal/setup/base.go b/internal/setup/base.go index ddf8e0fad..333c01736 100644 --- a/internal/setup/base.go +++ b/internal/setup/base.go @@ -44,6 +44,8 @@ import ( federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" fsinthttp "github.com/matrix-org/dendrite/federationsender/inthttp" "github.com/matrix-org/dendrite/internal/config" + keyserverAPI "github.com/matrix-org/dendrite/keyserver/api" + keyinthttp "github.com/matrix-org/dendrite/keyserver/inthttp" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" rsinthttp "github.com/matrix-org/dendrite/roomserver/inthttp" serverKeyAPI "github.com/matrix-org/dendrite/serverkeyapi/api" @@ -214,6 +216,15 @@ func (b *BaseDendrite) ServerKeyAPIClient() serverKeyAPI.ServerKeyInternalAPI { return f } +// KeyServerHTTPClient returns KeyInternalAPI for hitting the key server over HTTP +func (b *BaseDendrite) KeyServerHTTPClient() keyserverAPI.KeyInternalAPI { + f, err := keyinthttp.NewKeyServerClient(b.Cfg.KeyServerURL(), b.httpClient) + if err != nil { + logrus.WithError(err).Panic("KeyServerHTTPClient failed", b.httpClient) + } + return f +} + // CreateDeviceDB creates a new instance of the device database. Should only be // called once per component. func (b *BaseDendrite) CreateDeviceDB() devices.Database { diff --git a/internal/setup/monolith.go b/internal/setup/monolith.go index 6ace53a99..9ae629485 100644 --- a/internal/setup/monolith.go +++ b/internal/setup/monolith.go @@ -26,7 +26,7 @@ import ( federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/transactions" - "github.com/matrix-org/dendrite/keyserver" + keyAPI "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/mediaapi" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" serverKeyAPI "github.com/matrix-org/dendrite/serverkeyapi/api" @@ -56,6 +56,7 @@ type Monolith struct { ServerKeyAPI serverKeyAPI.ServerKeyInternalAPI UserAPI userapi.UserInternalAPI StateAPI currentstateAPI.CurrentStateInternalAPI + KeyAPI keyAPI.KeyInternalAPI // Optional ExtPublicRoomsProvider api.ExtraPublicRoomsProvider @@ -69,8 +70,6 @@ func (m *Monolith) AddAllPublicRoutes(publicMux *mux.Router) { m.EDUInternalAPI, m.AppserviceAPI, m.StateAPI, transactions.New(), m.FederationSenderAPI, m.UserAPI, m.ExtPublicRoomsProvider, ) - - keyserver.AddPublicRoutes(publicMux, m.Config, m.UserAPI) federationapi.AddPublicRoutes( publicMux, m.Config, m.UserAPI, m.FedClient, m.KeyRing, m.RoomserverAPI, m.FederationSenderAPI, diff --git a/keyserver/README.md b/keyserver/README.md new file mode 100644 index 000000000..fd9f37d27 --- /dev/null +++ b/keyserver/README.md @@ -0,0 +1,19 @@ +## Key Server + +This is an internal component which manages E2E keys from clients. It handles all the [Key Management APIs](https://matrix.org/docs/spec/client_server/r0.6.1#key-management-api) with the exception of `/keys/changes` which is handled by Sync API. This component is designed to shard by user ID. + +Keys are uploaded and stored in this component, and key changes are emitted to a Kafka topic for downstream components such as Sync API. + +### Internal APIs +- `PerformUploadKeys` stores identity keys and one-time public keys for given user(s). +- `PerformClaimKeys` acquires one-time public keys for given user(s). This may involve outbound federation calls. +- `QueryKeys` returns identity keys for given user(s). This may involve outbound federation calls. This component may then cache federated identity keys to avoid repeatedly hitting remote servers. +- A topic which emits identity keys every time there is a change (addition or deletion). + +### Endpoint mappings +- Client API maps `/keys/upload` to `PerformUploadKeys`. +- Client API maps `/keys/query` to `QueryKeys`. +- Client API maps `/keys/claim` to `PerformClaimKeys`. +- Federation API maps `/user/keys/query` to `QueryKeys`. +- Federation API maps `/user/keys/claim` to `PerformClaimKeys`. +- Sync API maps `/keys/changes` to consuming from the Kafka topic. diff --git a/keyserver/api/api.go b/keyserver/api/api.go new file mode 100644 index 000000000..d8b866f3a --- /dev/null +++ b/keyserver/api/api.go @@ -0,0 +1,49 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import "context" + +type KeyInternalAPI interface { + PerformUploadKeys(ctx context.Context, req *PerformUploadKeysRequest, res *PerformUploadKeysResponse) + PerformClaimKeys(ctx context.Context, req *PerformClaimKeysRequest, res *PerformClaimKeysResponse) + QueryKeys(ctx context.Context, req *QueryKeysRequest, res *QueryKeysResponse) +} + +// KeyError is returned if there was a problem performing/querying the server +type KeyError struct { + Error string +} + +type PerformUploadKeysRequest struct { +} + +type PerformUploadKeysResponse struct { + Error *KeyError +} + +type PerformClaimKeysRequest struct { +} + +type PerformClaimKeysResponse struct { + Error *KeyError +} + +type QueryKeysRequest struct { +} + +type QueryKeysResponse struct { + Error *KeyError +} diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go new file mode 100644 index 000000000..40883cdd6 --- /dev/null +++ b/keyserver/internal/internal.go @@ -0,0 +1,19 @@ +package internal + +import ( + "context" + + "github.com/matrix-org/dendrite/keyserver/api" +) + +type KeyInternalAPI struct{} + +func (a *KeyInternalAPI) PerformUploadKeys(ctx context.Context, req *api.PerformUploadKeysRequest, res *api.PerformUploadKeysResponse) { + +} +func (a *KeyInternalAPI) PerformClaimKeys(ctx context.Context, req *api.PerformClaimKeysRequest, res *api.PerformClaimKeysResponse) { + +} +func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysRequest, res *api.QueryKeysResponse) { + +} diff --git a/keyserver/inthttp/client.go b/keyserver/inthttp/client.go new file mode 100644 index 000000000..f2d00c705 --- /dev/null +++ b/keyserver/inthttp/client.go @@ -0,0 +1,103 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inthttp + +import ( + "context" + "errors" + "net/http" + + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/keyserver/api" + "github.com/opentracing/opentracing-go" +) + +// HTTP paths for the internal HTTP APIs +const ( + PerformUploadKeysPath = "/keyserver/performUploadKeys" + PerformClaimKeysPath = "/keyserver/performClaimKeys" + QueryKeysPath = "/keyserver/queryKeys" +) + +// NewKeyServerClient creates a KeyInternalAPI implemented by talking to a HTTP POST API. +// If httpClient is nil an error is returned +func NewKeyServerClient( + apiURL string, + httpClient *http.Client, +) (api.KeyInternalAPI, error) { + if httpClient == nil { + return nil, errors.New("NewKeyServerClient: httpClient is ") + } + return &httpKeyInternalAPI{ + apiURL: apiURL, + httpClient: httpClient, + }, nil +} + +type httpKeyInternalAPI struct { + apiURL string + httpClient *http.Client +} + +func (h *httpKeyInternalAPI) PerformClaimKeys( + ctx context.Context, + request *api.PerformClaimKeysRequest, + response *api.PerformClaimKeysResponse, +) { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformClaimKeys") + defer span.Finish() + + apiURL := h.apiURL + PerformClaimKeysPath + err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + if err != nil { + response.Error = &api.KeyError{ + Error: err.Error(), + } + } +} + +func (h *httpKeyInternalAPI) PerformUploadKeys( + ctx context.Context, + request *api.PerformUploadKeysRequest, + response *api.PerformUploadKeysResponse, +) { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformUploadKeys") + defer span.Finish() + + apiURL := h.apiURL + PerformUploadKeysPath + err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + if err != nil { + response.Error = &api.KeyError{ + Error: err.Error(), + } + } +} + +func (h *httpKeyInternalAPI) QueryKeys( + ctx context.Context, + request *api.QueryKeysRequest, + response *api.QueryKeysResponse, +) { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryKeys") + defer span.Finish() + + apiURL := h.apiURL + QueryKeysPath + err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + if err != nil { + response.Error = &api.KeyError{ + Error: err.Error(), + } + } +} diff --git a/keyserver/inthttp/server.go b/keyserver/inthttp/server.go new file mode 100644 index 000000000..ec78b6132 --- /dev/null +++ b/keyserver/inthttp/server.go @@ -0,0 +1,61 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inthttp + +import ( + "encoding/json" + "net/http" + + "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/keyserver/api" + "github.com/matrix-org/util" +) + +func AddRoutes(internalAPIMux *mux.Router, s api.KeyInternalAPI) { + internalAPIMux.Handle(PerformClaimKeysPath, + httputil.MakeInternalAPI("performClaimKeys", func(req *http.Request) util.JSONResponse { + request := api.PerformClaimKeysRequest{} + response := api.PerformClaimKeysResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + s.PerformClaimKeys(req.Context(), &request, &response) + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle(PerformUploadKeysPath, + httputil.MakeInternalAPI("performUploadKeys", func(req *http.Request) util.JSONResponse { + request := api.PerformUploadKeysRequest{} + response := api.PerformUploadKeysResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + s.PerformUploadKeys(req.Context(), &request, &response) + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle(QueryKeysPath, + httputil.MakeInternalAPI("queryKeys", func(req *http.Request) util.JSONResponse { + request := api.QueryKeysRequest{} + response := api.QueryKeysResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + s.QueryKeys(req.Context(), &request, &response) + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) +} diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go index bedc4dfb8..3bb0e462c 100644 --- a/keyserver/keyserver.go +++ b/keyserver/keyserver.go @@ -16,14 +16,19 @@ package keyserver import ( "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/internal/config" - "github.com/matrix-org/dendrite/keyserver/routing" - userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/keyserver/api" + "github.com/matrix-org/dendrite/keyserver/internal" + "github.com/matrix-org/dendrite/keyserver/inthttp" ) -// AddPublicRoutes registers HTTP handlers for CS API calls -func AddPublicRoutes( - router *mux.Router, cfg *config.Dendrite, userAPI userapi.UserInternalAPI, -) { - routing.Setup(router, cfg, userAPI) +// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions +// on the given input API. +func AddInternalRoutes(router *mux.Router, intAPI api.KeyInternalAPI) { + inthttp.AddRoutes(router, intAPI) +} + +// NewInternalAPI returns a concerete implementation of the internal API. Callers +// can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. +func NewInternalAPI() api.KeyInternalAPI { + return &internal.KeyInternalAPI{} } diff --git a/keyserver/routing/routing.go b/keyserver/routing/routing.go deleted file mode 100644 index dba43528f..000000000 --- a/keyserver/routing/routing.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package routing - -import ( - "net/http" - - "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/internal/config" - "github.com/matrix-org/dendrite/internal/httputil" - userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/util" -) - -const pathPrefixR0 = "/client/r0" - -// Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client -// to clients which need to make outbound HTTP requests. -// -// Due to Setup being used to call many other functions, a gocyclo nolint is -// applied: -// nolint: gocyclo -func Setup( - publicAPIMux *mux.Router, cfg *config.Dendrite, userAPI userapi.UserInternalAPI, -) { - r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter() - - r0mux.Handle("/keys/query", - httputil.MakeAuthAPI("queryKeys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - return QueryKeys(req) - }), - ).Methods(http.MethodPost, http.MethodOptions) - - r0mux.Handle("/keys/upload/{keyID}", - httputil.MakeAuthAPI("keys_upload", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - return util.JSONResponse{ - Code: 200, - JSON: map[string]interface{}{}, - } - }), - ).Methods(http.MethodPost, http.MethodOptions) -}