From 084ad07d16ff0d1cfe26e82be58c6749b117691e Mon Sep 17 00:00:00 2001 From: terrill <314156936@qq.com> Date: Mon, 2 Jul 2018 12:45:13 +0800 Subject: [PATCH 01/18] standardize format --- dendrite-config.yaml | 2 + docker/postgres/create_db.sh | 2 +- .../clientapi/auth/authtypes/device.go | 1 + .../dendrite/common/config/config.go | 2 + .../dendrite/encryptoapi/encryptoapi.go | 44 ++ .../dendrite/encryptoapi/routing/keys.go | 405 ++++++++++++++++++ .../dendrite/encryptoapi/routing/routing.go | 71 +++ .../encryptoapi/storage/encrypt_algorithm.go | 85 ++++ .../encryptoapi/storage/encrypt_keys_table.go | 192 +++++++++ .../dendrite/encryptoapi/storage/storage.go | 112 +++++ .../dendrite/encryptoapi/types/claim.go | 54 +++ .../dendrite/encryptoapi/types/query.go | 76 ++++ .../dendrite/encryptoapi/types/storage.go | 30 ++ .../dendrite/encryptoapi/types/upload.go | 150 +++++++ 14 files changed, 1225 insertions(+), 1 deletion(-) create mode 100644 src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go create mode 100644 src/github.com/matrix-org/dendrite/encryptoapi/routing/keys.go create mode 100644 src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go create mode 100644 src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_algorithm.go create mode 100644 src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go create mode 100644 src/github.com/matrix-org/dendrite/encryptoapi/storage/storage.go create mode 100644 src/github.com/matrix-org/dendrite/encryptoapi/types/claim.go create mode 100644 src/github.com/matrix-org/dendrite/encryptoapi/types/query.go create mode 100644 src/github.com/matrix-org/dendrite/encryptoapi/types/storage.go create mode 100644 src/github.com/matrix-org/dendrite/encryptoapi/types/upload.go diff --git a/dendrite-config.yaml b/dendrite-config.yaml index ae926bab8..82d7aae51 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -87,6 +87,7 @@ kafka: output_room_event: roomserverOutput output_client_data: clientapiOutput user_updates: userUpdates + keyUpdate: keyUpdate # The postgres connection configs for connecting to the databases e.g a postgres:// URI database: @@ -98,6 +99,7 @@ database: server_key: "postgres://dendrite:itsasecret@localhost/dendrite_serverkey?sslmode=disable" federation_sender: "postgres://dendrite:itsasecret@localhost/dendrite_federationsender?sslmode=disable" public_rooms_api: "postgres://dendrite:itsasecret@localhost/dendrite_publicroomsapi?sslmode=disable" + encrypt_api: "postgres:////dendrite:itsasecret@localhost/dendrite_encryptapi?sslmode=disable" # If using naffka you need to specify a naffka database # naffka: "postgres://dendrite:itsasecret@localhost/dendrite_naffka?sslmode=disable" diff --git a/docker/postgres/create_db.sh b/docker/postgres/create_db.sh index 56f6540ec..5835bde71 100644 --- a/docker/postgres/create_db.sh +++ b/docker/postgres/create_db.sh @@ -1,5 +1,5 @@ #!/bin/bash -for db in account device mediaapi syncapi roomserver serverkey federationsender publicroomsapi naffka; do +for db in account device mediaapi syncapi roomserver serverkey federationsender publicroomsapi naffka encryptapi; do createdb -O dendrite dendrite_$db done diff --git a/src/github.com/matrix-org/dendrite/clientapi/auth/authtypes/device.go b/src/github.com/matrix-org/dendrite/clientapi/auth/authtypes/device.go index a6d3a7b08..470e7615e 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/auth/authtypes/device.go +++ b/src/github.com/matrix-org/dendrite/clientapi/auth/authtypes/device.go @@ -22,4 +22,5 @@ type Device struct { // This uniquely identifies the device from all other devices and clients. AccessToken string // TODO: display name, last used timestamp, keys, etc + DisplayName string } diff --git a/src/github.com/matrix-org/dendrite/common/config/config.go b/src/github.com/matrix-org/dendrite/common/config/config.go index 8bbac80c6..769739d29 100644 --- a/src/github.com/matrix-org/dendrite/common/config/config.go +++ b/src/github.com/matrix-org/dendrite/common/config/config.go @@ -167,6 +167,8 @@ type Dendrite struct { PublicRoomsAPI DataSource `yaml:"public_rooms_api"` // The Naffka database is used internally by the naffka library, if used. Naffka DataSource `yaml:"naffka,omitempty"` + // Encryption api database + EncryptAPI DataSource `yaml:"encrypt_api"` } `yaml:"database"` // TURN Server Config diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go b/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go new file mode 100644 index 000000000..249c9649d --- /dev/null +++ b/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go @@ -0,0 +1,44 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package encryptoapi + +import ( + "github.com/matrix-org/dendrite/common/basecomponent" + "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" + "github.com/matrix-org/dendrite/encryptoapi/routing" + "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" + "github.com/matrix-org/dendrite/encryptoapi/storage" + "fmt" +) + +func SetupEcryptoapi( + base *basecomponent.BaseDendrite, + accountsDB *accounts.Database, + deviceDB *devices.Database, + +) { + + encryptionDB, err := storage.NewDatabase(string(base.Cfg.Database.EncryptAPI)) + fmt.Print(err) + routing.Setup( + base.APIMux, + *base.Cfg, + encryptionDB, + accountsDB, + deviceDB, + ) + routing.InitNotifier(base) + +} diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/routing/keys.go b/src/github.com/matrix-org/dendrite/encryptoapi/routing/keys.go new file mode 100644 index 000000000..84ede1153 --- /dev/null +++ b/src/github.com/matrix-org/dendrite/encryptoapi/routing/keys.go @@ -0,0 +1,405 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "github.com/matrix-org/util" + "github.com/matrix-org/dendrite/encryptoapi/storage" + "net/http" + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/encryptoapi/types" + "context" + "github.com/pkg/errors" + "strings" + "fmt" + "encoding/json" + "time" + "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/common/basecomponent" + "github.com/Shopify/sarama" +) + +const ( + TYPESUM = iota + TYPECLAIM + TYPEVAL + BODYDEVICEKEY + BODYONETIMEKEY + ONETIMEKEYSTRING + ONETIMEKEYOBJECT +) + +type KeyNotifier struct { + base *basecomponent.BaseDendrite + ch sarama.AsyncProducer +} + +var keyProducer = &KeyNotifier{} + +func UploadPKeys(req *http.Request, encryptionDB *storage.Database, userID, deviceID string) util.JSONResponse { + var keybody types.UploadEncrypt + if reqErr := httputil.UnmarshalJSONRequest(req, &keybody); reqErr != nil { + return *reqErr + } + keySpecific := turnSpecific(keybody) + err := persistKeys(encryptionDB, req.Context(), &keySpecific, userID, deviceID) + numMap := (QueryOneTimeKeys( + TYPESUM, + userID, + deviceID, + encryptionDB, + req.Context())).(map[string]int) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadGateway, + JSON: types.UploadResponse{ + Count: numMap, + }, + } + } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: types.UploadResponse{ + Count: numMap, + }, + } +} + +func QueryPKeys(req *http.Request, encryptionDB *storage.Database, userID, deviceID string, deviceDB *devices.Database) util.JSONResponse { + var queryRq types.QueryRequest + queryRp := types.QueryResponse{} + queryRp.Failure = make(map[string]interface{}) + queryRp.DeviceKeys = make(map[string]map[string]types.DeviceKeysQuery) + if reqErr := httputil.UnmarshalJSONRequest(req, &queryRq); reqErr != nil { + return *reqErr + } + + /* + federation consideration: when user id is in federation, a query is needed to ask fed for keys + domain --------+ fed (keys) + domain +--tout-- timer + */ + // todo: Add federation processing at specific userID. + if false /*federation judgement*/ { + tout := queryRq.Timeout + if tout == 0 { + tout = int64(10 * time.Second) + } + stimuCh := make(chan int) + go func() { + time.Sleep(time.Duration(tout) * 1000 * 1000) + close(stimuCh) + }() + select { + case <-stimuCh: + queryRp.Failure = make(map[string]interface{}) + // todo: key in this map is restricted to username at the end, yet a mocked one. + queryRp.Failure["@alice:localhost"] = "ran out of offered time" + } + } + + for uid, arr := range queryRq.DeviceKeys { + queryRp.DeviceKeys[uid] = make(map[string]types.DeviceKeysQuery) + deviceKeysQueryMap := queryRp.DeviceKeys[uid] + // backward compatible to old interface + midArr := []string{} + for device, _ := range arr.(map[string]interface{}) { + midArr = append(midArr, device) + } + dkeys, _ := encryptionDB.QueryInRange(req.Context(), uid, midArr) + for _, key := range dkeys { + // preset for complicated nested map struct + if _, ok := deviceKeysQueryMap[key.Device_id]; !ok { + // make consistency + deviceKeysQueryMap[key.Device_id] = types.DeviceKeysQuery{} + } + if deviceKeysQueryMap[key.Device_id].Signature == nil { + mid := make(map[string]map[string]string) + midmap := deviceKeysQueryMap[key.Device_id] + midmap.Signature = mid + deviceKeysQueryMap[key.Device_id] = midmap + } + if deviceKeysQueryMap[key.Device_id].Keys == nil { + mid := make(map[string]string) + midmap := deviceKeysQueryMap[key.Device_id] + midmap.Keys = mid + deviceKeysQueryMap[key.Device_id] = midmap + } + if _, ok := deviceKeysQueryMap[key.Device_id].Signature[uid]; !ok { + // make consistency + deviceKeysQueryMap[key.Device_id].Signature[uid] = make(map[string]string) + } + // load for accomplishment + single := deviceKeysQueryMap[key.Device_id] + + resKey := fmt.Sprintf("@%s:%s", key.Key_algorithm, key.Device_id) + resBody := key.Key + if _, ok := single.Keys[resKey]; !ok { + } + single.Keys[resKey] = resBody + single.DeviceId = key.Device_id + single.UserId = key.User_id + single.Signature[uid][fmt.Sprintf("@%s:%s", "ed25519", key.Device_id)] = key.Signature + single.Algorithm, _ = takeAL(*encryptionDB, req.Context(), key.User_id, key.Device_id) + localpart, _, _ := gomatrixserverlib.SplitID('@', uid) + device, _ := deviceDB.GetDeviceByID(req.Context(), localpart, deviceID) + single.Unsigned.Info = device.DisplayName + deviceKeysQueryMap[key.Device_id] = single + } + } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: queryRp, + } +} + +func ClaimOneTimeKeys(req *http.Request, encryptionDB *storage.Database, userID, deviceID string, deviceDB *devices.Database) util.JSONResponse { + var claimRq types.ClaimRequest + claimRp := types.ClaimResponse{} + claimRp.Failures = make(map[string]interface{}) + claimRp.ClaimBody = make(map[string]map[string]map[string]interface{}) + if reqErr := httputil.UnmarshalJSONRequest(req, &claimRq); reqErr != nil { + return *reqErr + } + + /* + federation consideration: when user id is in federation, a query is needed to ask fed for keys + domain --------+ fed (keys) + domain +--tout-- timer + */ + // todo: Add federation processing at specific userID. + if false /*federation judgement*/ { + tout := claimRq.Timeout + stimuCh := make(chan int) + go func() { + time.Sleep(time.Duration(tout) * 1000 * 1000) + close(stimuCh) + }() + select { + case <-stimuCh: + claimRp.Failures = make(map[string]interface{}) + // todo: key in this map is restricted to username at the end, yet a mocked one. + claimRp.Failures["@alice:localhost"] = "ran out of offered time" + } + } + + content := claimRq.ClaimDetail + for uid, detail := range content { + for deviceID, al := range detail { + var alTyp int + if strings.Contains(al, "signed") { + alTyp = ONETIMEKEYOBJECT + } else { + alTyp = ONETIMEKEYSTRING + } + key, err := pickOne(*encryptionDB, req.Context(), uid, deviceID, al) + if err != nil { + claimRp.Failures[uid] = fmt.Sprintf("%s:%s", "fail to get keys for device ", deviceID) + } + claimRp.ClaimBody[uid] = make(map[string]map[string]interface{}) + keymap := claimRp.ClaimBody[uid][deviceID] + keymap = make(map[string]interface{}) + switch alTyp { + case ONETIMEKEYSTRING: + keymap[fmt.Sprintf("%s:%s", al, key.Key_id)] = key.Key + case ONETIMEKEYOBJECT: + sig := make(map[string]map[string]string) + sig[uid] = make(map[string]string) + sig[uid][fmt.Sprintf("%s:%s", "ed25519", deviceID)] = key.Signature + keymap[fmt.Sprintf("%s:%s", al, key.Key_id)] = types.KeyObject{Key: key.Key, Signature: sig} + } + claimRp.ClaimBody[uid][deviceID] = keymap + } + } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: claimRp, + } +} + +func LookUpChangedPKeys() util.JSONResponse { + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } +} + +// todo: check through interface for duplicate +func checkUpload(req *types.UploadEncryptSpecific, typ int) bool { + if typ == BODYDEVICEKEY { + devicekey := req.DeviceKeys + if devicekey.UserId == "" { + return false + } + } + if typ == BODYONETIMEKEY { + if req.OneTimeKey.KeyString == nil || req.OneTimeKey.KeyObject == nil { + return false + } + } + return true +} + +// todo: complete this field through claim type +func QueryOneTimeKeys( + typ int, + userID, deviceID string, + encryptionDB *storage.Database, + ctx context.Context, +) interface{} { + if typ == TYPESUM { + res, _ := encryptionDB.SelectOneTimeKeyCount(ctx, deviceID, userID) + return res + } + return nil +} + +// todo: complete this function and invoke through sign out extension or some scenarios else those matter +// when web client sign out, a clean should be processed, cause all keys would never been used from then on. +func ClearUnused() {} + +func persistKeys( + database *storage.Database, + ctx context.Context, + body *types.UploadEncryptSpecific, + userID, + deviceID string, +) (err error) { + // in order to persist keys , a check filtering duplicate should be processed + if checkUpload(body, BODYDEVICEKEY) { + deviceKeys := body.DeviceKeys + al := deviceKeys.Algorithm + err = persistAl(*database, ctx, userID, deviceID, al) + if checkUpload(body, BODYONETIMEKEY) { + // insert one time keys firstly + onetimeKeys := body.OneTimeKey + for al_keyID, val := range onetimeKeys.KeyString { + al := (strings.Split(al_keyID, ":"))[0] + keyID := (strings.Split(al_keyID, ":"))[1] + keyInfo := val + keyTyp := "one_time_key" + sig := "" + database.InsertKey(ctx, deviceID, userID, keyID, keyTyp, keyInfo, al, sig) + } + for al_keyID, val := range onetimeKeys.KeyObject { + al := (strings.Split(al_keyID, ":"))[0] + keyID := (strings.Split(al_keyID, ":"))[1] + keyInfo := val.Key + keyTyp := "one_time_key" + sig := val.Signature[userID][fmt.Sprintf("%s:%s", "ed25519", deviceID)] + database.InsertKey(ctx, deviceID, userID, keyID, keyTyp, keyInfo, al, sig) + } + // insert device keys + keys := deviceKeys.Keys + sigs := deviceKeys.Signature + for al_device, key := range keys { + al := (strings.Split(al_device, ":"))[0] + keyTyp := "device_key" + keyInfo := key + keyID := "" + sig := sigs[userID][fmt.Sprintf("%s:%s", "ed25519", deviceID)] + database.InsertKey( + ctx, deviceID, userID, keyID, keyTyp, keyInfo, al, sig) + } + } else { + keys := deviceKeys.Keys + sigs := deviceKeys.Signature + for al_device, key := range keys { + al := (strings.Split(al_device, ":"))[0] + keyTyp := "device_key" + keyInfo := key + keyID := "" + sig := sigs[userID][fmt.Sprintf("%s:%s", "ed25519", deviceID)] + database.InsertKey(ctx, deviceID, userID, keyID, keyTyp, keyInfo, al, sig) + } + } + // notifier to sync server + upnotify(userID) + } else { + if checkUpload(body, BODYONETIMEKEY) { + onetimeKeys := body.OneTimeKey + for al_keyID, val := range onetimeKeys.KeyString { + al := (strings.Split(al_keyID, ":"))[0] + keyID := (strings.Split(al_keyID, ":"))[1] + keyInfo := val + keyTyp := "one_time_key" + sig := "" + database.InsertKey(ctx, deviceID, userID, keyID, keyTyp, keyInfo, al, sig) + } + for al_keyID, val := range onetimeKeys.KeyObject { + al := (strings.Split(al_keyID, ":"))[0] + keyID := (strings.Split(al_keyID, ":"))[1] + keyInfo := val.Key + keyTyp := "one_time_key" + sig := val.Signature[userID][fmt.Sprintf("%s:%s", "ed25519", deviceID)] + database.InsertKey(ctx, deviceID, userID, keyID, keyTyp, keyInfo, al, sig) + } + } else { + return errors.New("Fail to touch keys !") + } + } + return err +} + +func turnSpecific(cont types.UploadEncrypt) (spec types.UploadEncryptSpecific) { + // both device keys are coordinate + spec.DeviceKeys = cont.DeviceKeys + spec.OneTimeKey.KeyString = make(map[string]string) + spec.OneTimeKey.KeyObject = make(map[string]types.KeyObject) + mapStringInterface := cont.OneTimeKey + for key, val := range mapStringInterface { + value, ok := val.(string) + if ok { + spec.OneTimeKey.KeyString[key] = value + } else { + valueObject := types.KeyObject{} + target, _ := json.Marshal(val) + json.Unmarshal(target, &valueObject) + spec.OneTimeKey.KeyObject[key] = valueObject + } + } + return +} + +func persistAl(encryptDB storage.Database, ctx context.Context, uid, device string, al []string) (err error) { + err = encryptDB.InsertAl(ctx, uid, device, al) + return +} + +func takeAL(encryptDB storage.Database, ctx context.Context, uid, device string) (al []string, err error) { + al, err = encryptDB.SelectAl(ctx, uid, device) + return +} + +func pickOne(encryptDB storage.Database, ctx context.Context, uid, device, al string) (key types.KeyHolder, err error) { + key, err = encryptDB.SelectOneTimeKeySingle(ctx, uid, device, al) + return +} + +func upnotify(userID string) { + m := sarama.ProducerMessage{ + Topic: "keyUpdate", + Key: sarama.StringEncoder("key"), + Value: sarama.StringEncoder(userID), + } + keyProducer.ch.Input() <- &m +} + +func InitNotifier(base *basecomponent.BaseDendrite) { + keyProducer.base = base + pro, _ := sarama.NewAsyncProducer(base.Cfg.Kafka.Addresses, nil) + keyProducer.ch = pro +} diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go b/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go new file mode 100644 index 000000000..38fc10922 --- /dev/null +++ b/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go @@ -0,0 +1,71 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "net/http" + + "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" + "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/encryptoapi/storage" + "github.com/matrix-org/util" +) + +const pathPrefixR0 = "/_matrix/client/r0" +const pathPrefixUnstable = "/_matrix/client/unstable" + +func Setup( + apiMux *mux.Router, + cfg config.Dendrite, + encryptionDB *storage.Database, + accountDB *accounts.Database, + deviceDB *devices.Database, +) { + //r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() + unstablemux := apiMux.PathPrefix(pathPrefixUnstable).Subrouter() + + unstablemux.Handle("/keys/upload/{deviceID}", + common.MakeAuthAPI("upload keys", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + return UploadPKeys(req, encryptionDB, device.UserID, device.ID) + }), + ).Methods(http.MethodPost, http.MethodOptions) + + unstablemux.Handle("/keys/upload", + common.MakeAuthAPI("upload keys", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + return UploadPKeys(req, encryptionDB, device.UserID, device.ID) + }), + ).Methods(http.MethodPost, http.MethodOptions) + + unstablemux.Handle("/keys/query", + common.MakeAuthAPI("query keys", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + //vars := mux.Vars(req) + return QueryPKeys(req, encryptionDB, device.UserID, device.ID, deviceDB) + }), + ).Methods(http.MethodPost, http.MethodOptions) + + unstablemux.Handle("/keys/claim", + common.MakeAuthAPI("claim keys", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + //vars := mux.Vars(req) + return ClaimOneTimeKeys(req, encryptionDB, device.UserID, device.ID, deviceDB) + }), + ).Methods(http.MethodPost, http.MethodOptions) + + +} diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_algorithm.go b/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_algorithm.go new file mode 100644 index 000000000..9c03e0f26 --- /dev/null +++ b/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_algorithm.go @@ -0,0 +1,85 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "database/sql" + "context" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/encryptoapi/types" +) + +const algorithmSchema = ` +-- The media_repository table holds metadata for each media file stored and accessible to the local server, +-- the actual file is stored separately. +CREATE TABLE IF NOT EXISTS encrypt_algorithm ( + device_id TEXT NOT NULL, + user_id TEXT NOT NULL, + algorithms TEXT NOT NULL +); +` + +const insertalSQL = ` +INSERT INTO encrypt_algorithm (device_id, user_id, algorithms) VALUES ($1, $2, $3) +` + +const selectalSQL = ` +SELECT user_id, device_id, algorithms FROM encrypt_algorithm WHERE user_id = $1 AND device_id = $2 +` + +type alStatements struct { + insertAlStmt *sql.Stmt + selectAlStmt *sql.Stmt +} + +func (s *alStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(algorithmSchema) + if err != nil { + return + } + if s.insertAlStmt, err = db.Prepare(insertalSQL); err != nil { + return + } + if s.selectAlStmt, err = db.Prepare(selectalSQL); err != nil { + return + } + return +} + +func (ks *alStatements) insertAl( + ctx context.Context, txn *sql.Tx, + userID, deviceID, algorithms string, +) error { + stmt := common.TxStmt(txn, ks.insertAlStmt) + _, err := stmt.ExecContext(ctx, deviceID, userID, algorithms) + return err +} + +func (ks *alStatements) selectAl( + ctx context.Context, + txn *sql.Tx, + userID, deviceID string, +) (holder types.AlHolder, err error) { + + stmt := common.TxStmt(txn, ks.selectAlStmt) + row := stmt.QueryRowContext(ctx, userID, deviceID) + single := types.AlHolder{} + err = row.Scan( + &single.User_id, + &single.Device_id, + &single.Supported_algorithm, + ) + return single, err +} diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go b/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go new file mode 100644 index 000000000..79bb9de84 --- /dev/null +++ b/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go @@ -0,0 +1,192 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "database/sql" + "context" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/encryptoapi/types" + "github.com/lib/pq" +) + +const keysSchema = ` +-- The media_repository table holds metadata for each media file stored and accessible to the local server, +-- the actual file is stored separately. +CREATE TABLE IF NOT EXISTS encrypt_keys ( + device_id TEXT NOT NULL, + user_id TEXT NOT NULL, + key_id TEXT , + key_type TEXT NOT NULL, + key_info TEXT NOT NULL, + algorithm TEXT NOT NULL, + signature TEXT NOT NULL +); +` + +const insertkeySQL = ` +INSERT INTO encrypt_keys (device_id, user_id, key_id, key_type, key_info, algorithm, signature) + VALUES ($1, $2, $3, $4, $5, $6, $7) +` + +const selectkeySQL = ` +SELECT user_id, device_id, key_id, key_type, key_info, algorithm, signature FROM encrypt_keys WHERE user_id = $1 AND device_id = $2 +` + +const deleteSinglekeySQL = ` +SELECT user_id, device_id, key_id, key_type, key_info, algorithm, signature FROM encrypt_keys WHERE user_id = $1 AND device_id = $2 AND algorithm = $3 +` +const selectSinglekeySQL = ` +DELETE FROM encrypt_keys WHERE user_id = $1 AND device_id = $2 AND algorithm = $3 AND key_id = $4 +` + +const selectInkeysSQL = ` +SELECT user_id, device_id, key_id, key_type, key_info, algorithm, signature FROM encrypt_keys WHERE user_id = $1 AND key_type = 'device_key' AND device_id = ANY($2) +` + +const selectAllkeysSQL = ` +SELECT user_id, device_id, key_id, key_type, key_info, algorithm, signature FROM encrypt_keys WHERE user_id = $1 AND key_type = $2 +` + +type keyStatements struct { + insertKeyStmt *sql.Stmt + selectKeyStmt *sql.Stmt + selectInKeysStmt *sql.Stmt + selectAllKeyStmt *sql.Stmt + selectSingleKeyStmt *sql.Stmt + deleteSingleKeyStmt *sql.Stmt +} + +func (s *keyStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(keysSchema) + if err != nil { + return + } + if s.insertKeyStmt, err = db.Prepare(insertkeySQL); err != nil { + return + } + if s.selectKeyStmt, err = db.Prepare(selectkeySQL); err != nil { + return + } + if s.selectInKeysStmt, err = db.Prepare(selectInkeysSQL); err != nil { + return + } + if s.selectAllKeyStmt, err = db.Prepare(selectAllkeysSQL); err != nil { + return + } + if s.deleteSingleKeyStmt, err = db.Prepare(selectSinglekeySQL); err != nil { + return + } + if s.selectSingleKeyStmt, err = db.Prepare(deleteSinglekeySQL); err != nil { + return + } + return +} + +func (ks *keyStatements) insertKey( + ctx context.Context, txn *sql.Tx, + deviceID, userID, keyID, keyTyp, keyInfo, algorithm, signature string, +) error { + stmt := common.TxStmt(txn, ks.insertKeyStmt) + _, err := stmt.ExecContext(ctx, deviceID, userID, keyID, keyTyp, keyInfo, algorithm, signature) + return err +} + +func (ks *keyStatements) selectKey( + ctx context.Context, + txn *sql.Tx, + deviceID, userID string, +) (holders []types.KeyHolder, err error) { + stmt := common.TxStmt(txn, ks.selectKeyStmt) + rows, err := stmt.QueryContext(ctx, userID, deviceID) + if err != nil { + return nil, err + } + for rows.Next() { + single := &types.KeyHolder{} + if err := rows.Scan( + &single.User_id, + &single.Device_id, + &single.Key_id, + &single.Key_type, + &single.Key, + &single.Key_algorithm, + &single.Signature, + ); err != nil { + return nil, err + } + holders = append(holders, *single) + } + return holders, err +} +func (ks *keyStatements) selectSingleKey( + ctx context.Context, + userID, deviceID, algorithm string, +) (holder types.KeyHolder, err error) { + stmt := ks.selectSingleKeyStmt + row := stmt.QueryRowContext(ctx, userID, deviceID, algorithm) + if err != nil { + return holder, err + } + if err := row.Scan( + &holder.User_id, + &holder.Device_id, + &holder.Key_id, + &holder.Key_type, + &holder.Key, + &holder.Key_algorithm, + &holder.Signature, + ); err != nil { + deleteStmt := ks.deleteSingleKeyStmt + deleteStmt.ExecContext(ctx, userID, deviceID, algorithm, holder.Key_id) + return holder, err + } + return holder, err +} + +func (ks *keyStatements) selectInKeys( + ctx context.Context, + userID string, + arr []string, +) (holders []types.KeyHolder, err error) { + rows := &sql.Rows{} + stmt := ks.selectAllKeyStmt + if len(arr) == 0 { + rows, err = stmt.QueryContext(ctx, userID, "device_key") + } else { + stmt = ks.selectInKeysStmt + list := pq.Array(arr) + rows, err = stmt.QueryContext(ctx, userID, list) + } + if err != nil { + return nil, err + } + for rows.Next() { + single := &types.KeyHolder{} + if err := rows.Scan( + &single.User_id, + &single.Device_id, + &single.Key_id, + &single.Key_type, + &single.Key, + &single.Key_algorithm, + &single.Signature, + ); err != nil { + return nil, err + } + holders = append(holders, *single) + } + return holders, err +} diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/storage/storage.go b/src/github.com/matrix-org/dendrite/encryptoapi/storage/storage.go new file mode 100644 index 000000000..ea58b7e82 --- /dev/null +++ b/src/github.com/matrix-org/dendrite/encryptoapi/storage/storage.go @@ -0,0 +1,112 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "database/sql" + "github.com/matrix-org/dendrite/common" + "context" + "github.com/matrix-org/dendrite/encryptoapi/types" + "strings" +) + +// Database represents a presence database. +type Database struct { + db *sql.DB + keyStatements keyStatements + alStatements alStatements +} + +// NewDatabase creates a new presence database +func NewDatabase(dataSourceName string) (*Database, error) { + var db *sql.DB + var err error + if db, err = sql.Open("postgres_hook", dataSourceName); err != nil { + return nil, err + } + keyStatement := keyStatements{} + alStatement := alStatements{} + if err = keyStatement.prepare(db); err != nil { + return nil, err + } + if err = alStatement.prepare(db); err != nil { + return nil, err + } + return &Database{db: db, keyStatements: keyStatement, alStatements: alStatement}, nil +} + +func (d *Database) InsertKey( + ctx context.Context, + deviceID, userID, keyID, keyTyp, keyInfo, al, sig string, +) (err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + return d.keyStatements.insertKey(ctx, txn, deviceID, userID, keyID, keyTyp, keyInfo, al, sig) + }) + return +} + +func (d *Database) SelectOneTimeKeyCount( + ctx context.Context, + deviceID, userID string, +) (m map[string]int, err error) { + m = make(map[string]int) + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + elems, err := d.keyStatements.selectKey(ctx, txn, deviceID, userID) + for _, val := range elems { + if _, ok := m[val.Key_algorithm]; !ok { + m[val.Key_algorithm] = 0 + } + if val.Key_type == "one_time_key" { + m[val.Key_algorithm] += 1 + } + } + return err + }) + return +} + +func (d *Database) QueryInRange( + ctx context.Context, + userID string, + arr []string, +) (res []types.KeyHolder, err error) { + res, err = d.keyStatements.selectInKeys(ctx, userID, arr) + return +} + +func (d *Database) InsertAl(ctx context.Context, uid, device string, al []string) (err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) (err error) { + d.alStatements.insertAl(ctx, txn, uid, device, strings.Join(al, ",")) + return + }) + return +} + +func (d *Database) SelectAl(ctx context.Context, uid, device string) (res []string, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) (err error) { + holder, err := d.alStatements.selectAl(ctx, txn, uid, device) + res = strings.Split(holder.Supported_algorithm, ",") + return + }) + return +} + +func (d *Database) SelectOneTimeKeySingle( + ctx context.Context, + userID, deviceID, algorithm string, +) (holder types.KeyHolder, err error) { + holder, err = d.keyStatements.selectSingleKey(ctx, userID, deviceID, algorithm) + return +} diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/types/claim.go b/src/github.com/matrix-org/dendrite/encryptoapi/types/claim.go new file mode 100644 index 000000000..36dba5f8f --- /dev/null +++ b/src/github.com/matrix-org/dendrite/encryptoapi/types/claim.go @@ -0,0 +1,54 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +/* + { + "timeout": 10000, + "one_time_keys": { + "@alice:example.com": { + "JLAFKJWSCS": "curve25519" + } + } + } +*/ +type ClaimRequest struct { + Timeout int64 `json:"timeout"` + ClaimDetail map[string]map[string]string `json:"one_time_keys"` +} + +/* + { + "failures": {}, + "one_time_keys": { + "@alice:example.com": { + "JLAFKJWSCS": { + "signed_curve25519:AAAAHg": { + "key": "zKbLg+NrIjpnagy+pIY6uPL4ZwEG2v+8F9lmgsnlZzs", + "signatures": { + "@alice:example.com": { + "ed25519:JLAFKJWSCS": "FLWxXqGbwrb8SM3Y795eB6OA8bwBcoMZFXBqnTn58AYWZSqiD45tlBVcDa2L7RwdKXebW/VzDlnfVJ+9jok1Bw" + } + } + } + } + } + } + } +*/ +type ClaimResponse struct { + Failures map[string]interface{} `json:"failures"` + ClaimBody map[string]map[string]map[string]interface{} `json:"one_time_keys"` +} diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/types/query.go b/src/github.com/matrix-org/dendrite/encryptoapi/types/query.go new file mode 100644 index 000000000..fed20ec53 --- /dev/null +++ b/src/github.com/matrix-org/dendrite/encryptoapi/types/query.go @@ -0,0 +1,76 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +/* + { + "timeout": 10000, + "device_keys": { + "@alice:example.com": ["DISYYYX","XYIISONM"] + } + } +*/ +type QueryRequest struct { + Timeout int64 `json:"timeout"` + DeviceKeys map[string]interface{} `json:"device_keys"` + Token string `json:"token"` +} + +/* + { + "failures": {}, + "device_keys": { + "@alice:example.com": { + "JLAFKJWSCS": { + "user_id": "@alice:example.com", + "device_id": "JLAFKJWSCS", + "algorithms": [ + "m.olm.curve25519-aes-sha256", + "m.megolm.v1.aes-sha" + ], + "keys": { + "curve25519:JLAFKJWSCS": "3C5BFWi2Y8MaVvjM8M22DBmh24PmgR0nPvJOIArzgyI", + "ed25519:JLAFKJWSCS": "lEuiRJBit0IG6nUf5pUzWTUEsRVVe/HJkoKuEww9ULI" + }, + "signatures": { + "@alice:example.com": { + "ed25519:JLAFKJWSCS": "dSO80A01XiigH3uBiDVx/EjzaoycHcjq9lfQX0uWsqxl2giMIiSPR8a4d291W1ihKJL/a+myXS367WT6NAIcBA" + } + }, + "unsigned": { + "device_display_name": "Alice's mobile phone" + } + } + } + } + } +*/ +type QueryResponse struct { + Failure map[string]interface{} `json:"failures"` + DeviceKeys map[string]map[string]DeviceKeysQuery `json:"device_keys"` +} + +type DeviceKeysQuery struct { + UserId string `json:"user_id"` + DeviceId string `json:"device_id"` + Algorithm []string `json:"algorithms"` + Keys map[string]string `json:"keys"` + Signature map[string]map[string]string `json:"signatures"` + Unsigned UnsignedDeviceInfo `json:"unsigned"` +} + +type UnsignedDeviceInfo struct { + Info string `json:"device_display_name"` +} diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/types/storage.go b/src/github.com/matrix-org/dendrite/encryptoapi/types/storage.go new file mode 100644 index 000000000..9dc44eb75 --- /dev/null +++ b/src/github.com/matrix-org/dendrite/encryptoapi/types/storage.go @@ -0,0 +1,30 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +type KeyHolder struct { + User_id, + Device_id, + Signature, key, + Key_algorithm, + Key_id, + Key, + Key_type string +} +type AlHolder struct { + User_id, + Device_id, + Supported_algorithm string +} diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/types/upload.go b/src/github.com/matrix-org/dendrite/encryptoapi/types/upload.go new file mode 100644 index 000000000..3b4267d1c --- /dev/null +++ b/src/github.com/matrix-org/dendrite/encryptoapi/types/upload.go @@ -0,0 +1,150 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +/* + { + "device_keys": { + "user_id": "@alice:example.com", + "device_id": "JLAFKJWSCS", + "algorithms": [ + "m.olm.curve25519-aes-sha256", + "m.megolm.v1.aes-sha" + ], + "keys": { + "curve25519:JLAFKJWSCS": "3C5BFWi2Y8MaVvjM8M22DBmh24PmgR0nPvJOIArzgyI", + "ed25519:JLAFKJWSCS": "lEuiRJBit0IG6nUf5pUzWTUEsRVVe/HJkoKuEww9ULI" + }, + "signatures": { + "@alice:example.com": { + "ed25519:JLAFKJWSCS": "dSO80A01XiigH3uBiDVx/EjzaoycHcjq9lfQX0uWsqxl2giMIiSPR8a4d291W1ihKJL/a+myXS367WT6NAIcBA" + } + } + }, + "one_time_keys": { + "curve25519:AAAAAQ": "/qyvZvwjiTxGdGU0RCguDCLeR+nmsb3FfNG3/Ve4vU8", + "signed_curve25519:AAAAHg": { + "key": "zKbLg+NrIjpnagy+pIY6uPL4ZwEG2v+8F9lmgsnlZzs", + "signatures": { + "@alice:example.com": { + "ed25519:JLAFKJWSCS": "FLWxXqGbwrb8SM3Y795eB6OA8bwBcoMZFXBqnTn58AYWZSqiD45tlBVcDa2L7RwdKXebW/VzDlnfVJ+9jok1Bw" + } + } + }, + "signed_curve25519:AAAAHQ": { + "key": "j3fR3HemM16M7CWhoI4Sk5ZsdmdfQHsKL1xuSft6MSw", + "signatures": { + "@alice:example.com": { + "ed25519:JLAFKJWSCS": "IQeCEPb9HFk217cU9kw9EOiusC6kMIkoIRnbnfOh5Oc63S1ghgyjShBGpu34blQomoalCyXWyhaaT3MrLZYQAA" + } + } + } + } + } +*/ +type UploadEncrypt struct { + DeviceKeys DeviceKeys `json:"device_keys"` + OneTimeKey map[string]interface{} `json:"one_time_keys"` +} +type UploadEncryptSpecific struct { + DeviceKeys DeviceKeys `json:"device_keys"` + OneTimeKey OneTimeKeySpecific `json:"one_time_keys"` +} + +/* + { + "one_time_key_counts": { + "curve25519": 10, + "signed_curve25519": 20 + } + } +*/ +type UploadResponse struct { + Count map[string]int `json:"one_time_key_counts"` +} + +/* + "device_keys": { + "user_id": "@alice:example.com", + "device_id": "JLAFKJWSCS", + "algorithms": [ + "m.olm.curve25519-aes-sha256", + "m.megolm.v1.aes-sha" + ], + "keys": { + "curve25519:JLAFKJWSCS": "3C5BFWi2Y8MaVvjM8M22DBmh24PmgR0nPvJOIArzgyI", + "ed25519:JLAFKJWSCS": "lEuiRJBit0IG6nUf5pUzWTUEsRVVe/HJkoKuEww9ULI" + }, + "signatures": { + "@alice:example.com": { + "ed25519:JLAFKJWSCS": "dSO80A01XiigH3uBiDVx/EjzaoycHcjq9lfQX0uWsqxl2giMIiSPR8a4d291W1ihKJL/a+myXS367WT6NAIcBA" + } + } + } +*/ +type DeviceKeys struct { + UserId string `json:"user_id"` + DeviceId string `json:"device_id"` + Algorithm []string `json:"algorithms"` + Keys map[string]string `json:"keys"` + Signature map[string]map[string]string `json:"signatures"` +} + +/* + "signed_curve25519:AAAAHg": { + "key": "zKbLg+NrIjpnagy+pIY6uPL4ZwEG2v+8F9lmgsnlZzs", + "signatures": { + "@alice:example.com": { + "ed25519:JLAFKJWSCS": "FLWxXqGbwrb8SM3Y795eB6OA8bwBcoMZFXBqnTn58AYWZSqiD45tlBVcDa2L7RwdKXebW/VzDlnfVJ+9jok1Bw" + } + } + } +*/ +type KeyObject struct { + Key string `json:"key"` + Signature map[string]map[string]string `json:"signatures"` +} + +/* + "one_time_keys": { + "curve25519:AAAAAQ": "/qyvZvwjiTxGdGU0RCguDCLeR+nmsb3FfNG3/Ve4vU8", + "signed_curve25519:AAAAHg": { + "key": "zKbLg+NrIjpnagy+pIY6uPL4ZwEG2v+8F9lmgsnlZzs", + "signatures": { + "@alice:example.com": { + "ed25519:JLAFKJWSCS": "FLWxXqGbwrb8SM3Y795eB6OA8bwBcoMZFXBqnTn58AYWZSqiD45tlBVcDa2L7RwdKXebW/VzDlnfVJ+9jok1Bw" + } + } + }, + "signed_curve25519:AAAAHQ": { + "key": "j3fR3HemM16M7CWhoI4Sk5ZsdmdfQHsKL1xuSft6MSw", + "signatures": { + "@alice:example.com": { + "ed25519:JLAFKJWSCS": "IQeCEPb9HFk217cU9kw9EOiusC6kMIkoIRnbnfOh5Oc63S1ghgyjShBGpu34blQomoalCyXWyhaaT3MrLZYQAA" + } + } + } + } +*/ +type OneTimeKey struct { + //KeyString map[string]string + //KeyObject map[string]KeyObject + KeySth map[string]interface{} +} +type OneTimeKeySpecific struct { + KeyString map[string]string + KeyObject map[string]KeyObject + //KeySth map[string]interface{} +} From e5effc328cf4b9c6dc270df5afad9c9872b65236 Mon Sep 17 00:00:00 2001 From: terrill <314156936@qq.com> Date: Tue, 3 Jul 2018 10:00:52 +0800 Subject: [PATCH 02/18] standardize format and spacing and codes length --- .../dendrite/encryptoapi/encryptoapi.go | 2 +- .../dendrite/encryptoapi/routing/keys.go | 64 +++++++++++++++---- .../dendrite/encryptoapi/routing/routing.go | 2 +- .../encryptoapi/storage/encrypt_algorithm.go | 10 +-- .../encryptoapi/storage/encrypt_keys_table.go | 24 +++---- .../dendrite/encryptoapi/storage/storage.go | 10 ++- .../dendrite/encryptoapi/types/claim.go | 2 +- .../dendrite/encryptoapi/types/query.go | 2 +- .../dendrite/encryptoapi/types/storage.go | 2 +- .../dendrite/encryptoapi/types/upload.go | 2 +- 10 files changed, 80 insertions(+), 40 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go b/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go index 249c9649d..b103dc827 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2018 Vector Creations Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/routing/keys.go b/src/github.com/matrix-org/dendrite/encryptoapi/routing/keys.go index 84ede1153..11ab6acdf 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/routing/keys.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/routing/keys.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2018 Vector Creations Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -49,13 +49,22 @@ type KeyNotifier struct { var keyProducer = &KeyNotifier{} -func UploadPKeys(req *http.Request, encryptionDB *storage.Database, userID, deviceID string) util.JSONResponse { +func UploadPKeys( + req *http.Request, + encryptionDB *storage.Database, + userID, deviceID string, +) util.JSONResponse { var keybody types.UploadEncrypt - if reqErr := httputil.UnmarshalJSONRequest(req, &keybody); reqErr != nil { + if reqErr := httputil.UnmarshalJSONRequest(req, &keybody); + reqErr != nil { return *reqErr } keySpecific := turnSpecific(keybody) - err := persistKeys(encryptionDB, req.Context(), &keySpecific, userID, deviceID) + err := persistKeys( + encryptionDB, + req.Context(), + &keySpecific, + userID, deviceID) numMap := (QueryOneTimeKeys( TYPESUM, userID, @@ -78,17 +87,24 @@ func UploadPKeys(req *http.Request, encryptionDB *storage.Database, userID, devi } } -func QueryPKeys(req *http.Request, encryptionDB *storage.Database, userID, deviceID string, deviceDB *devices.Database) util.JSONResponse { +func QueryPKeys( + req *http.Request, + encryptionDB *storage.Database, + userID, deviceID string, + deviceDB *devices.Database, +) util.JSONResponse { var queryRq types.QueryRequest queryRp := types.QueryResponse{} queryRp.Failure = make(map[string]interface{}) queryRp.DeviceKeys = make(map[string]map[string]types.DeviceKeysQuery) - if reqErr := httputil.UnmarshalJSONRequest(req, &queryRq); reqErr != nil { + if reqErr := httputil.UnmarshalJSONRequest(req, &queryRq); + reqErr != nil { return *reqErr } /* - federation consideration: when user id is in federation, a query is needed to ask fed for keys + federation consideration: when user id is in federation, a + query is needed to ask fed for keys. domain --------+ fed (keys) domain +--tout-- timer */ @@ -166,7 +182,12 @@ func QueryPKeys(req *http.Request, encryptionDB *storage.Database, userID, devic } } -func ClaimOneTimeKeys(req *http.Request, encryptionDB *storage.Database, userID, deviceID string, deviceDB *devices.Database) util.JSONResponse { +func ClaimOneTimeKeys( + req *http.Request, + encryptionDB *storage.Database, + userID, deviceID string, + deviceDB *devices.Database, +) util.JSONResponse { var claimRq types.ClaimRequest claimRp := types.ClaimResponse{} claimRp.Failures = make(map[string]interface{}) @@ -354,7 +375,9 @@ func persistKeys( return err } -func turnSpecific(cont types.UploadEncrypt) (spec types.UploadEncryptSpecific) { +func turnSpecific( + cont types.UploadEncrypt, +) (spec types.UploadEncryptSpecific) { // both device keys are coordinate spec.DeviceKeys = cont.DeviceKeys spec.OneTimeKey.KeyString = make(map[string]string) @@ -374,19 +397,32 @@ func turnSpecific(cont types.UploadEncrypt) (spec types.UploadEncryptSpecific) { return } -func persistAl(encryptDB storage.Database, ctx context.Context, uid, device string, al []string) (err error) { +func persistAl( + encryptDB storage.Database, + ctx context.Context, + uid, device string, + al []string, +) (err error) { err = encryptDB.InsertAl(ctx, uid, device, al) return } -func takeAL(encryptDB storage.Database, ctx context.Context, uid, device string) (al []string, err error) { +func takeAL( + encryptDB storage.Database, + ctx context.Context, + uid, device string, +) (al []string, err error) { al, err = encryptDB.SelectAl(ctx, uid, device) return } -func pickOne(encryptDB storage.Database, ctx context.Context, uid, device, al string) (key types.KeyHolder, err error) { - key, err = encryptDB.SelectOneTimeKeySingle(ctx, uid, device, al) - return +func pickOne( + encryptDB storage.Database, + ctx context.Context, + uid, device, al string, +) (key types.KeyHolder, err error) { +key, err = encryptDB.SelectOneTimeKeySingle(ctx, uid, device, al) +return } func upnotify(userID string) { diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go b/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go index 38fc10922..a8a5eb592 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2018 Vector Creations Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_algorithm.go b/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_algorithm.go index 9c03e0f26..18657e859 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_algorithm.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_algorithm.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2018 Vector Creations Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -30,13 +30,13 @@ CREATE TABLE IF NOT EXISTS encrypt_algorithm ( algorithms TEXT NOT NULL ); ` - const insertalSQL = ` -INSERT INTO encrypt_algorithm (device_id, user_id, algorithms) VALUES ($1, $2, $3) +INSERT INTO encrypt_algorithm (device_id, user_id, algorithms) +VALUES ($1, $2, $3) ` - const selectalSQL = ` -SELECT user_id, device_id, algorithms FROM encrypt_algorithm WHERE user_id = $1 AND device_id = $2 +SELECT user_id, device_id, algorithms FROM encrypt_algorithm +WHERE user_id = $1 AND device_id = $2 ` type alStatements struct { diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go b/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go index 79bb9de84..fdce52d9e 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2018 Vector Creations Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -35,29 +35,29 @@ CREATE TABLE IF NOT EXISTS encrypt_keys ( signature TEXT NOT NULL ); ` - const insertkeySQL = ` INSERT INTO encrypt_keys (device_id, user_id, key_id, key_type, key_info, algorithm, signature) - VALUES ($1, $2, $3, $4, $5, $6, $7) +VALUES ($1, $2, $3, $4, $5, $6, $7) ` - const selectkeySQL = ` -SELECT user_id, device_id, key_id, key_type, key_info, algorithm, signature FROM encrypt_keys WHERE user_id = $1 AND device_id = $2 +SELECT user_id, device_id, key_id, key_type, key_info, algorithm, signature FROM encrypt_keys +WHERE user_id = $1 AND device_id = $2 ` - const deleteSinglekeySQL = ` -SELECT user_id, device_id, key_id, key_type, key_info, algorithm, signature FROM encrypt_keys WHERE user_id = $1 AND device_id = $2 AND algorithm = $3 +SELECT user_id, device_id, key_id, key_type, key_info, algorithm, signature FROM encrypt_keys +WHERE user_id = $1 AND device_id = $2 AND algorithm = $3 ` const selectSinglekeySQL = ` -DELETE FROM encrypt_keys WHERE user_id = $1 AND device_id = $2 AND algorithm = $3 AND key_id = $4 +DELETE FROM encrypt_keys +WHERE user_id = $1 AND device_id = $2 AND algorithm = $3 AND key_id = $4 ` - const selectInkeysSQL = ` -SELECT user_id, device_id, key_id, key_type, key_info, algorithm, signature FROM encrypt_keys WHERE user_id = $1 AND key_type = 'device_key' AND device_id = ANY($2) +SELECT user_id, device_id, key_id, key_type, key_info, algorithm, signature FROM encrypt_keys + WHERE user_id = $1 AND key_type = 'device_key' AND device_id = ANY($2) ` - const selectAllkeysSQL = ` -SELECT user_id, device_id, key_id, key_type, key_info, algorithm, signature FROM encrypt_keys WHERE user_id = $1 AND key_type = $2 +SELECT user_id, device_id, key_id, key_type, key_info, algorithm, signature FROM encrypt_keys +WHERE user_id = $1 AND key_type = $2 ` type keyStatements struct { diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/storage/storage.go b/src/github.com/matrix-org/dendrite/encryptoapi/storage/storage.go index ea58b7e82..bb14bb2ab 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/storage/storage.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/storage/storage.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2018 Vector Creations Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -86,7 +86,9 @@ func (d *Database) QueryInRange( return } -func (d *Database) InsertAl(ctx context.Context, uid, device string, al []string) (err error) { +func (d *Database) InsertAl( + ctx context.Context, uid, device string, al []string, +) (err error) { err = common.WithTransaction(d.db, func(txn *sql.Tx) (err error) { d.alStatements.insertAl(ctx, txn, uid, device, strings.Join(al, ",")) return @@ -94,7 +96,9 @@ func (d *Database) InsertAl(ctx context.Context, uid, device string, al []string return } -func (d *Database) SelectAl(ctx context.Context, uid, device string) (res []string, err error) { +func (d *Database) SelectAl( + ctx context.Context, uid, device string, +) (res []string, err error) { err = common.WithTransaction(d.db, func(txn *sql.Tx) (err error) { holder, err := d.alStatements.selectAl(ctx, txn, uid, device) res = strings.Split(holder.Supported_algorithm, ",") diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/types/claim.go b/src/github.com/matrix-org/dendrite/encryptoapi/types/claim.go index 36dba5f8f..80a786d20 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/types/claim.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/types/claim.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2018 Vector Creations Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/types/query.go b/src/github.com/matrix-org/dendrite/encryptoapi/types/query.go index fed20ec53..1c5dba3a2 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/types/query.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/types/query.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2018 Vector Creations Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/types/storage.go b/src/github.com/matrix-org/dendrite/encryptoapi/types/storage.go index 9dc44eb75..55815420a 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/types/storage.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/types/storage.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2018 Vector Creations Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/types/upload.go b/src/github.com/matrix-org/dendrite/encryptoapi/types/upload.go index 3b4267d1c..b2f542075 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/types/upload.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/types/upload.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2018 Vector Creations Ltd // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. From 40955a378a4bb7c38beb48c267cdc8c81da7200b Mon Sep 17 00:00:00 2001 From: terrill <314156936@qq.com> Date: Tue, 3 Jul 2018 10:38:18 +0800 Subject: [PATCH 03/18] add blurbs and comment for nested code and each important function --- .../dendrite/encryptoapi/encryptoapi.go | 4 ++++ .../dendrite/encryptoapi/routing/keys.go | 19 ++++++++++++++++++- .../encryptoapi/storage/encrypt_algorithm.go | 2 ++ .../encryptoapi/storage/encrypt_keys_table.go | 5 +++++ .../dendrite/encryptoapi/storage/storage.go | 6 ++++++ 5 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go b/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go index b103dc827..dfb746d85 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go @@ -23,6 +23,10 @@ import ( "fmt" ) +// in order to gain key management capability +// , CMD should involve this invoke into main function +// , a setup need an assemble of i.e configs as base and +// accountDB and deviceDB func SetupEcryptoapi( base *basecomponent.BaseDendrite, accountsDB *accounts.Database, diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/routing/keys.go b/src/github.com/matrix-org/dendrite/encryptoapi/routing/keys.go index 11ab6acdf..6a1889219 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/routing/keys.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/routing/keys.go @@ -49,6 +49,8 @@ type KeyNotifier struct { var keyProducer = &KeyNotifier{} +// this function is for user upload his device key, and one-time-key +// to a limit at 50 set as default func UploadPKeys( req *http.Request, encryptionDB *storage.Database, @@ -60,11 +62,13 @@ func UploadPKeys( return *reqErr } keySpecific := turnSpecific(keybody) + // persist keys into encryptionDB err := persistKeys( encryptionDB, req.Context(), &keySpecific, userID, deviceID) + // numMap is algorithm-num map numMap := (QueryOneTimeKeys( TYPESUM, userID, @@ -87,6 +91,7 @@ func UploadPKeys( } } +// this function is for user query other's device key func QueryPKeys( req *http.Request, encryptionDB *storage.Database, @@ -127,15 +132,19 @@ func QueryPKeys( } } + // query one's device key from user corresponding to uid for uid, arr := range queryRq.DeviceKeys { queryRp.DeviceKeys[uid] = make(map[string]types.DeviceKeysQuery) deviceKeysQueryMap := queryRp.DeviceKeys[uid] // backward compatible to old interface midArr := []string{} + // figure out device list from devices described as device which is actually deviceID for device, _ := range arr.(map[string]interface{}) { midArr = append(midArr, device) } + // all device keys dkeys, _ := encryptionDB.QueryInRange(req.Context(), uid, midArr) + // build response for them for _, key := range dkeys { // preset for complicated nested map struct if _, ok := deviceKeysQueryMap[key.Device_id]; !ok { @@ -182,6 +191,7 @@ func QueryPKeys( } } +// claim for one time key that may be used in session exchange in olm encryption func ClaimOneTimeKeys( req *http.Request, encryptionDB *storage.Database, @@ -258,7 +268,8 @@ func LookUpChangedPKeys() util.JSONResponse { } } -// todo: check through interface for duplicate +// todo: check through interface for duplicate and what type of request should it be +// whether device or one time or both of them func checkUpload(req *types.UploadEncryptSpecific, typ int) bool { if typ == BODYDEVICEKEY { devicekey := req.DeviceKeys @@ -292,6 +303,7 @@ func QueryOneTimeKeys( // when web client sign out, a clean should be processed, cause all keys would never been used from then on. func ClearUnused() {} +// persist both device keys and one time keys func persistKeys( database *storage.Database, ctx context.Context, @@ -300,6 +312,10 @@ func persistKeys( deviceID string, ) (err error) { // in order to persist keys , a check filtering duplicate should be processed + // true stands for counterparts are in request + // situation 1: only device keys + // situation 2: both device keys and one time keys + // situation 3: only one time keys if checkUpload(body, BODYDEVICEKEY) { deviceKeys := body.DeviceKeys al := deviceKeys.Algorithm @@ -375,6 +391,7 @@ func persistKeys( return err } +// make keys instantiated to specific struct from keybody interface{} func turnSpecific( cont types.UploadEncrypt, ) (spec types.UploadEncryptSpecific) { diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_algorithm.go b/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_algorithm.go index 18657e859..d243028c6 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_algorithm.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_algorithm.go @@ -58,6 +58,7 @@ func (s *alStatements) prepare(db *sql.DB) (err error) { return } +// persist algorithms func (ks *alStatements) insertAl( ctx context.Context, txn *sql.Tx, userID, deviceID, algorithms string, @@ -67,6 +68,7 @@ func (ks *alStatements) insertAl( return err } +// select algorithms func (ks *alStatements) selectAl( ctx context.Context, txn *sql.Tx, diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go b/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go index fdce52d9e..c314770af 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go @@ -95,6 +95,7 @@ func (s *keyStatements) prepare(db *sql.DB) (err error) { return } +// insert keys func (ks *keyStatements) insertKey( ctx context.Context, txn *sql.Tx, deviceID, userID, keyID, keyTyp, keyInfo, algorithm, signature string, @@ -104,6 +105,7 @@ func (ks *keyStatements) insertKey( return err } +// select by user and device func (ks *keyStatements) selectKey( ctx context.Context, txn *sql.Tx, @@ -131,6 +133,8 @@ func (ks *keyStatements) selectKey( } return holders, err } + +// select single one for claim usage func (ks *keyStatements) selectSingleKey( ctx context.Context, userID, deviceID, algorithm string, @@ -156,6 +160,7 @@ func (ks *keyStatements) selectSingleKey( return holder, err } +// select details by given an array of devices func (ks *keyStatements) selectInKeys( ctx context.Context, userID string, diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/storage/storage.go b/src/github.com/matrix-org/dendrite/encryptoapi/storage/storage.go index bb14bb2ab..a08f4ad87 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/storage/storage.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/storage/storage.go @@ -47,6 +47,7 @@ func NewDatabase(dataSourceName string) (*Database, error) { return &Database{db: db, keyStatements: keyStatement, alStatements: alStatement}, nil } +// insert device key func (d *Database) InsertKey( ctx context.Context, deviceID, userID, keyID, keyTyp, keyInfo, al, sig string, @@ -57,6 +58,7 @@ func (d *Database) InsertKey( return } +// for key upload response usage a map from key algorithm to sum to counterpart func (d *Database) SelectOneTimeKeyCount( ctx context.Context, deviceID, userID string, @@ -77,6 +79,7 @@ func (d *Database) SelectOneTimeKeyCount( return } +// query keys in a range of devices func (d *Database) QueryInRange( ctx context.Context, userID string, @@ -86,6 +89,7 @@ func (d *Database) QueryInRange( return } +// persist algorithms func (d *Database) InsertAl( ctx context.Context, uid, device string, al []string, ) (err error) { @@ -96,6 +100,7 @@ func (d *Database) InsertAl( return } +// select algorithms func (d *Database) SelectAl( ctx context.Context, uid, device string, ) (res []string, err error) { @@ -107,6 +112,7 @@ func (d *Database) SelectAl( return } +// claim for one time key one for once func (d *Database) SelectOneTimeKeySingle( ctx context.Context, userID, deviceID, algorithm string, From ba19d86d47abb647e26875ffbf3d49514ec73e63 Mon Sep 17 00:00:00 2001 From: terrill <314156936@qq.com> Date: Tue, 3 Jul 2018 10:40:31 +0800 Subject: [PATCH 04/18] add encryptoapi to main in monolith server --- .../matrix-org/dendrite/cmd/dendrite-monolith-server/main.go | 3 +++ src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go b/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go index 3ffc833ea..301de9c9b 100644 --- a/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go @@ -34,6 +34,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" + "github.com/matrix-org/dendrite/encryptoapi" ) var ( @@ -56,6 +57,8 @@ func main() { alias, input, query := roomserver.SetupRoomServerComponent(base) + encryptoapi.SetupEcryptoapi(base, accountDB, deviceDB) + clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, federation, &keyRing, alias, input, query, diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go b/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go index dfb746d85..77fd7dc46 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go @@ -31,7 +31,6 @@ func SetupEcryptoapi( base *basecomponent.BaseDendrite, accountsDB *accounts.Database, deviceDB *devices.Database, - ) { encryptionDB, err := storage.NewDatabase(string(base.Cfg.Database.EncryptAPI)) From 7326c1293f2f86d6082671fd37810b1bf3dfe650 Mon Sep 17 00:00:00 2001 From: terrill <314156936@qq.com> Date: Wed, 4 Jul 2018 16:36:49 +0800 Subject: [PATCH 05/18] config file bug fix --- dendrite-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dendrite-config.yaml b/dendrite-config.yaml index 82d7aae51..b371a2ab4 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -99,7 +99,7 @@ database: server_key: "postgres://dendrite:itsasecret@localhost/dendrite_serverkey?sslmode=disable" federation_sender: "postgres://dendrite:itsasecret@localhost/dendrite_federationsender?sslmode=disable" public_rooms_api: "postgres://dendrite:itsasecret@localhost/dendrite_publicroomsapi?sslmode=disable" - encrypt_api: "postgres:////dendrite:itsasecret@localhost/dendrite_encryptapi?sslmode=disable" + encrypt_api: "postgres://dendrite:itsasecret@localhost/dendrite_encryptapi?sslmode=disable" # If using naffka you need to specify a naffka database # naffka: "postgres://dendrite:itsasecret@localhost/dendrite_naffka?sslmode=disable" From b09d6801f7e072d9cd85a50c1ad27b56b72dcde5 Mon Sep 17 00:00:00 2001 From: terrill <314156936@qq.com> Date: Wed, 4 Jul 2018 18:47:40 +0800 Subject: [PATCH 06/18] db drive bug fix --- .../matrix-org/dendrite/encryptoapi/storage/storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/storage/storage.go b/src/github.com/matrix-org/dendrite/encryptoapi/storage/storage.go index a08f4ad87..40e4c1fe2 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/storage/storage.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/storage/storage.go @@ -33,7 +33,7 @@ type Database struct { func NewDatabase(dataSourceName string) (*Database, error) { var db *sql.DB var err error - if db, err = sql.Open("postgres_hook", dataSourceName); err != nil { + if db, err = sql.Open("postgres", dataSourceName); err != nil { return nil, err } keyStatement := keyStatements{} From 2e20f7bcab0930ba5ec9602209c85ef914994904 Mon Sep 17 00:00:00 2001 From: terrill <314156936@qq.com> Date: Thu, 5 Jul 2018 00:27:11 +0800 Subject: [PATCH 07/18] rows defer close , log with err --- .../matrix-org/dendrite/encryptoapi/encryptoapi.go | 8 ++++---- .../dendrite/encryptoapi/storage/encrypt_keys_table.go | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go b/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go index 77fd7dc46..e391ecc12 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go @@ -20,7 +20,7 @@ import ( "github.com/matrix-org/dendrite/encryptoapi/routing" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/encryptoapi/storage" - "fmt" + "github.com/sirupsen/logrus" ) // in order to gain key management capability @@ -32,9 +32,10 @@ func SetupEcryptoapi( accountsDB *accounts.Database, deviceDB *devices.Database, ) { - encryptionDB, err := storage.NewDatabase(string(base.Cfg.Database.EncryptAPI)) - fmt.Print(err) + if err != nil { + logrus.WithError(err).Panicf("failed to connect to encryption db") + } routing.Setup( base.APIMux, *base.Cfg, @@ -43,5 +44,4 @@ func SetupEcryptoapi( deviceDB, ) routing.InitNotifier(base) - } diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go b/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go index c314770af..c3c7ed469 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go @@ -113,6 +113,7 @@ func (ks *keyStatements) selectKey( ) (holders []types.KeyHolder, err error) { stmt := common.TxStmt(txn, ks.selectKeyStmt) rows, err := stmt.QueryContext(ctx, userID, deviceID) + defer rows.Close() if err != nil { return nil, err } @@ -167,6 +168,7 @@ func (ks *keyStatements) selectInKeys( arr []string, ) (holders []types.KeyHolder, err error) { rows := &sql.Rows{} + defer rows.Close() stmt := ks.selectAllKeyStmt if len(arr) == 0 { rows, err = stmt.QueryContext(ctx, userID, "device_key") From d97c71992cf449abc8e5b6cdeb7d5756eb9588f4 Mon Sep 17 00:00:00 2001 From: terrill <314156936@qq.com> Date: Thu, 5 Jul 2018 15:10:51 +0800 Subject: [PATCH 08/18] find lint revision in encryption api server codes --- .../cmd/dendrite-monolith-server/main.go | 4 +- .../dendrite/encryptoapi/encryptoapi.go | 10 +- .../dendrite/encryptoapi/routing/keys.go | 361 +++++++++++------- .../dendrite/encryptoapi/routing/routing.go | 13 +- .../encryptoapi/storage/encrypt_algorithm.go | 18 +- .../encryptoapi/storage/encrypt_keys_table.go | 76 ++-- .../dendrite/encryptoapi/storage/storage.go | 26 +- .../dendrite/encryptoapi/types/claim.go | 33 +- .../dendrite/encryptoapi/types/query.go | 49 +-- .../dendrite/encryptoapi/types/storage.go | 21 +- .../dendrite/encryptoapi/types/upload.go | 111 +----- 11 files changed, 324 insertions(+), 398 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go b/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go index 301de9c9b..e0433d748 100644 --- a/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go @@ -32,9 +32,9 @@ import ( "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/syncapi" + "github.com/matrix-org/dendrite/encryptoapi" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" - "github.com/matrix-org/dendrite/encryptoapi" ) var ( @@ -57,7 +57,7 @@ func main() { alias, input, query := roomserver.SetupRoomServerComponent(base) - encryptoapi.SetupEcryptoapi(base, accountDB, deviceDB) + encryptoapi.SetupEcryptoapi(base, deviceDB) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go b/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go index e391ecc12..d4529a27c 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go @@ -15,10 +15,9 @@ package encryptoapi import ( - "github.com/matrix-org/dendrite/common/basecomponent" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/encryptoapi/routing" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" + "github.com/matrix-org/dendrite/common/basecomponent" + "github.com/matrix-org/dendrite/encryptoapi/routing" "github.com/matrix-org/dendrite/encryptoapi/storage" "github.com/sirupsen/logrus" ) @@ -27,9 +26,10 @@ import ( // , CMD should involve this invoke into main function // , a setup need an assemble of i.e configs as base and // accountDB and deviceDB + +// SetupEcryptoapi set up to servers func SetupEcryptoapi( base *basecomponent.BaseDendrite, - accountsDB *accounts.Database, deviceDB *devices.Database, ) { encryptionDB, err := storage.NewDatabase(string(base.Cfg.Database.EncryptAPI)) @@ -38,9 +38,7 @@ func SetupEcryptoapi( } routing.Setup( base.APIMux, - *base.Cfg, encryptionDB, - accountsDB, deviceDB, ) routing.InitNotifier(base) diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/routing/keys.go b/src/github.com/matrix-org/dendrite/encryptoapi/routing/keys.go index 6a1889219..60a8bfa41 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/routing/keys.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/routing/keys.go @@ -15,33 +15,43 @@ package routing import ( - "github.com/matrix-org/util" - "github.com/matrix-org/dendrite/encryptoapi/storage" - "net/http" - "github.com/matrix-org/dendrite/clientapi/httputil" - "github.com/matrix-org/dendrite/encryptoapi/types" "context" - "github.com/pkg/errors" - "strings" - "fmt" "encoding/json" - "time" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/dendrite/common/basecomponent" + "fmt" "github.com/Shopify/sarama" + "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/common/basecomponent" + "github.com/matrix-org/dendrite/encryptoapi/storage" + "github.com/matrix-org/dendrite/encryptoapi/types" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/pkg/errors" + "net/http" + "strings" + "time" ) const ( - TYPESUM = iota - TYPECLAIM - TYPEVAL + // TYPESUM sum type + TYPESUM = iota + // BODYDEVICEKEY device key body BODYDEVICEKEY + // BODYONETIMEKEY one time key BODYONETIMEKEY + // ONETIMEKEYSTRING key string ONETIMEKEYSTRING + // ONETIMEKEYOBJECT key object ONETIMEKEYOBJECT ) +// ONETIMEKEYSTR stands for storage string property +const ONETIMEKEYSTR = "one_time_key" + +// DEVICEKEYSTR stands for storage string property +const DEVICEKEYSTR = "device_key" + +// KeyNotifier kafka notifier type KeyNotifier struct { base *basecomponent.BaseDendrite ch sarama.AsyncProducer @@ -49,32 +59,30 @@ type KeyNotifier struct { var keyProducer = &KeyNotifier{} -// this function is for user upload his device key, and one-time-key -// to a limit at 50 set as default +// UploadPKeys this function is for user upload his device key, and one-time-key to a limit at 50 set as default func UploadPKeys( req *http.Request, encryptionDB *storage.Database, userID, deviceID string, ) util.JSONResponse { var keybody types.UploadEncrypt - if reqErr := httputil.UnmarshalJSONRequest(req, &keybody); - reqErr != nil { + if reqErr := httputil.UnmarshalJSONRequest(req, &keybody); reqErr != nil { return *reqErr } keySpecific := turnSpecific(keybody) // persist keys into encryptionDB err := persistKeys( - encryptionDB, req.Context(), + encryptionDB, &keySpecific, userID, deviceID) // numMap is algorithm-num map numMap := (QueryOneTimeKeys( + req.Context(), TYPESUM, userID, deviceID, - encryptionDB, - req.Context())).(map[string]int) + encryptionDB)).(map[string]int) if err != nil { return util.JSONResponse{ Code: http.StatusBadGateway, @@ -91,19 +99,19 @@ func UploadPKeys( } } -// this function is for user query other's device key +// QueryPKeys this function is for user query other's device key func QueryPKeys( req *http.Request, encryptionDB *storage.Database, - userID, deviceID string, + deviceID string, deviceDB *devices.Database, ) util.JSONResponse { + var err error var queryRq types.QueryRequest queryRp := types.QueryResponse{} queryRp.Failure = make(map[string]interface{}) queryRp.DeviceKeys = make(map[string]map[string]types.DeviceKeysQuery) - if reqErr := httputil.UnmarshalJSONRequest(req, &queryRq); - reqErr != nil { + if reqErr := httputil.UnmarshalJSONRequest(req, &queryRq); reqErr != nil { return *reqErr } @@ -129,6 +137,8 @@ func QueryPKeys( queryRp.Failure = make(map[string]interface{}) // todo: key in this map is restricted to username at the end, yet a mocked one. queryRp.Failure["@alice:localhost"] = "ran out of offered time" + case <-make(chan interface{}): + // todo : here goes federation chan , still a mocked one } } @@ -139,50 +149,35 @@ func QueryPKeys( // backward compatible to old interface midArr := []string{} // figure out device list from devices described as device which is actually deviceID - for device, _ := range arr.(map[string]interface{}) { + for device := range arr.(map[string]interface{}) { midArr = append(midArr, device) } // all device keys dkeys, _ := encryptionDB.QueryInRange(req.Context(), uid, midArr) // build response for them - for _, key := range dkeys { - // preset for complicated nested map struct - if _, ok := deviceKeysQueryMap[key.Device_id]; !ok { - // make consistency - deviceKeysQueryMap[key.Device_id] = types.DeviceKeysQuery{} - } - if deviceKeysQueryMap[key.Device_id].Signature == nil { - mid := make(map[string]map[string]string) - midmap := deviceKeysQueryMap[key.Device_id] - midmap.Signature = mid - deviceKeysQueryMap[key.Device_id] = midmap - } - if deviceKeysQueryMap[key.Device_id].Keys == nil { - mid := make(map[string]string) - midmap := deviceKeysQueryMap[key.Device_id] - midmap.Keys = mid - deviceKeysQueryMap[key.Device_id] = midmap - } - if _, ok := deviceKeysQueryMap[key.Device_id].Signature[uid]; !ok { - // make consistency - deviceKeysQueryMap[key.Device_id].Signature[uid] = make(map[string]string) - } - // load for accomplishment - single := deviceKeysQueryMap[key.Device_id] - resKey := fmt.Sprintf("@%s:%s", key.Key_algorithm, key.Device_id) + for _, key := range dkeys { + + deviceKeysQueryMap = presetDeviceKeysQueryMap(deviceKeysQueryMap, uid, key) + // load for accomplishment + single := deviceKeysQueryMap[key.DeviceID] + resKey := fmt.Sprintf("@%s:%s", key.KeyAlgorithm, key.DeviceID) resBody := key.Key - if _, ok := single.Keys[resKey]; !ok { - } single.Keys[resKey] = resBody - single.DeviceId = key.Device_id - single.UserId = key.User_id - single.Signature[uid][fmt.Sprintf("@%s:%s", "ed25519", key.Device_id)] = key.Signature - single.Algorithm, _ = takeAL(*encryptionDB, req.Context(), key.User_id, key.Device_id) + single.DeviceID = key.DeviceID + single.UserID = key.UserID + single.Signature[uid][fmt.Sprintf("@%s:%s", "ed25519", key.DeviceID)] = key.Signature + single.Algorithm, err = takeAL(req.Context(), *encryptionDB, key.UserID, key.DeviceID) localpart, _, _ := gomatrixserverlib.SplitID('@', uid) device, _ := deviceDB.GetDeviceByID(req.Context(), localpart, deviceID) single.Unsigned.Info = device.DisplayName - deviceKeysQueryMap[key.Device_id] = single + deviceKeysQueryMap[key.DeviceID] = single + } + } + if err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: queryRp, } } return util.JSONResponse{ @@ -191,12 +186,10 @@ func QueryPKeys( } } -// claim for one time key that may be used in session exchange in olm encryption +// ClaimOneTimeKeys claim for one time key that may be used in session exchange in olm encryption func ClaimOneTimeKeys( req *http.Request, encryptionDB *storage.Database, - userID, deviceID string, - deviceDB *devices.Database, ) util.JSONResponse { var claimRq types.ClaimRequest claimRp := types.ClaimResponse{} @@ -224,6 +217,8 @@ func ClaimOneTimeKeys( claimRp.Failures = make(map[string]interface{}) // todo: key in this map is restricted to username at the end, yet a mocked one. claimRp.Failures["@alice:localhost"] = "ran out of offered time" + case <-make(chan interface{}): + // todo : here goes federation chan , still a mocked one } } @@ -236,21 +231,24 @@ func ClaimOneTimeKeys( } else { alTyp = ONETIMEKEYSTRING } - key, err := pickOne(*encryptionDB, req.Context(), uid, deviceID, al) + key, err := pickOne(req.Context(), *encryptionDB, uid, deviceID, al) if err != nil { claimRp.Failures[uid] = fmt.Sprintf("%s:%s", "fail to get keys for device ", deviceID) } claimRp.ClaimBody[uid] = make(map[string]map[string]interface{}) - keymap := claimRp.ClaimBody[uid][deviceID] - keymap = make(map[string]interface{}) + keyPreMap := claimRp.ClaimBody[uid] + keymap := keyPreMap[deviceID] + if keymap == nil { + keymap = make(map[string]interface{}) + } switch alTyp { case ONETIMEKEYSTRING: - keymap[fmt.Sprintf("%s:%s", al, key.Key_id)] = key.Key + keymap[fmt.Sprintf("%s:%s", al, key.KeyID)] = key.Key case ONETIMEKEYOBJECT: sig := make(map[string]map[string]string) sig[uid] = make(map[string]string) sig[uid][fmt.Sprintf("%s:%s", "ed25519", deviceID)] = key.Signature - keymap[fmt.Sprintf("%s:%s", al, key.Key_id)] = types.KeyObject{Key: key.Key, Signature: sig} + keymap[fmt.Sprintf("%s:%s", al, key.KeyID)] = types.KeyObject{Key: key.Key, Signature: sig} } claimRp.ClaimBody[uid][deviceID] = keymap } @@ -261,19 +259,12 @@ func ClaimOneTimeKeys( } } -func LookUpChangedPKeys() util.JSONResponse { - return util.JSONResponse{ - Code: http.StatusOK, - JSON: struct{}{}, - } -} - // todo: check through interface for duplicate and what type of request should it be // whether device or one time or both of them func checkUpload(req *types.UploadEncryptSpecific, typ int) bool { if typ == BODYDEVICEKEY { devicekey := req.DeviceKeys - if devicekey.UserId == "" { + if devicekey.UserID == "" { return false } } @@ -285,12 +276,12 @@ func checkUpload(req *types.UploadEncryptSpecific, typ int) bool { return true } -// todo: complete this field through claim type +// QueryOneTimeKeys todo: complete this field through claim type func QueryOneTimeKeys( + ctx context.Context, typ int, userID, deviceID string, encryptionDB *storage.Database, - ctx context.Context, ) interface{} { if typ == TYPESUM { res, _ := encryptionDB.SelectOneTimeKeyCount(ctx, deviceID, userID) @@ -299,14 +290,14 @@ func QueryOneTimeKeys( return nil } +// ClearUnused when web client sign out, a clean should be processed, cause all keys would never been used from then on. // todo: complete this function and invoke through sign out extension or some scenarios else those matter -// when web client sign out, a clean should be processed, cause all keys would never been used from then on. func ClearUnused() {} // persist both device keys and one time keys func persistKeys( - database *storage.Database, ctx context.Context, + database *storage.Database, body *types.UploadEncryptSpecific, userID, deviceID string, @@ -319,73 +310,28 @@ func persistKeys( if checkUpload(body, BODYDEVICEKEY) { deviceKeys := body.DeviceKeys al := deviceKeys.Algorithm - err = persistAl(*database, ctx, userID, deviceID, al) + err = persistAl(ctx, *database, userID, deviceID, al) + if err != nil { + return + } if checkUpload(body, BODYONETIMEKEY) { - // insert one time keys firstly - onetimeKeys := body.OneTimeKey - for al_keyID, val := range onetimeKeys.KeyString { - al := (strings.Split(al_keyID, ":"))[0] - keyID := (strings.Split(al_keyID, ":"))[1] - keyInfo := val - keyTyp := "one_time_key" - sig := "" - database.InsertKey(ctx, deviceID, userID, keyID, keyTyp, keyInfo, al, sig) - } - for al_keyID, val := range onetimeKeys.KeyObject { - al := (strings.Split(al_keyID, ":"))[0] - keyID := (strings.Split(al_keyID, ":"))[1] - keyInfo := val.Key - keyTyp := "one_time_key" - sig := val.Signature[userID][fmt.Sprintf("%s:%s", "ed25519", deviceID)] - database.InsertKey(ctx, deviceID, userID, keyID, keyTyp, keyInfo, al, sig) - } - // insert device keys - keys := deviceKeys.Keys - sigs := deviceKeys.Signature - for al_device, key := range keys { - al := (strings.Split(al_device, ":"))[0] - keyTyp := "device_key" - keyInfo := key - keyID := "" - sig := sigs[userID][fmt.Sprintf("%s:%s", "ed25519", deviceID)] - database.InsertKey( - ctx, deviceID, userID, keyID, keyTyp, keyInfo, al, sig) + if err = bothKeyProcess(ctx, body, userID, deviceID, database, deviceKeys); err != nil { + return } } else { - keys := deviceKeys.Keys - sigs := deviceKeys.Signature - for al_device, key := range keys { - al := (strings.Split(al_device, ":"))[0] - keyTyp := "device_key" - keyInfo := key - keyID := "" - sig := sigs[userID][fmt.Sprintf("%s:%s", "ed25519", deviceID)] - database.InsertKey(ctx, deviceID, userID, keyID, keyTyp, keyInfo, al, sig) + if err = dkeyProcess(ctx, userID, deviceID, database, deviceKeys); err != nil { + return } } // notifier to sync server upnotify(userID) } else { if checkUpload(body, BODYONETIMEKEY) { - onetimeKeys := body.OneTimeKey - for al_keyID, val := range onetimeKeys.KeyString { - al := (strings.Split(al_keyID, ":"))[0] - keyID := (strings.Split(al_keyID, ":"))[1] - keyInfo := val - keyTyp := "one_time_key" - sig := "" - database.InsertKey(ctx, deviceID, userID, keyID, keyTyp, keyInfo, al, sig) - } - for al_keyID, val := range onetimeKeys.KeyObject { - al := (strings.Split(al_keyID, ":"))[0] - keyID := (strings.Split(al_keyID, ":"))[1] - keyInfo := val.Key - keyTyp := "one_time_key" - sig := val.Signature[userID][fmt.Sprintf("%s:%s", "ed25519", deviceID)] - database.InsertKey(ctx, deviceID, userID, keyID, keyTyp, keyInfo, al, sig) + if err = otmKeyProcess(ctx, body, userID, deviceID, database); err != nil { + return } } else { - return errors.New("Fail to touch keys !") + return errors.New("failed to touch keys") } } return err @@ -407,7 +353,10 @@ func turnSpecific( } else { valueObject := types.KeyObject{} target, _ := json.Marshal(val) - json.Unmarshal(target, &valueObject) + err := json.Unmarshal(target, &valueObject) + if err != nil { + continue + } spec.OneTimeKey.KeyObject[key] = valueObject } } @@ -415,8 +364,8 @@ func turnSpecific( } func persistAl( - encryptDB storage.Database, ctx context.Context, + encryptDB storage.Database, uid, device string, al []string, ) (err error) { @@ -425,8 +374,8 @@ func persistAl( } func takeAL( - encryptDB storage.Database, ctx context.Context, + encryptDB storage.Database, uid, device string, ) (al []string, err error) { al, err = encryptDB.SelectAl(ctx, uid, device) @@ -434,12 +383,12 @@ func takeAL( } func pickOne( - encryptDB storage.Database, ctx context.Context, + encryptDB storage.Database, uid, device, al string, ) (key types.KeyHolder, err error) { -key, err = encryptDB.SelectOneTimeKeySingle(ctx, uid, device, al) -return + key, err = encryptDB.SelectOneTimeKeySingle(ctx, uid, device, al) + return } func upnotify(userID string) { @@ -451,8 +400,138 @@ func upnotify(userID string) { keyProducer.ch.Input() <- &m } +// InitNotifier initialize kafka notifier func InitNotifier(base *basecomponent.BaseDendrite) { keyProducer.base = base pro, _ := sarama.NewAsyncProducer(base.Cfg.Kafka.Addresses, nil) keyProducer.ch = pro } + +func presetDeviceKeysQueryMap( + deviceKeysQueryMap map[string]types.DeviceKeysQuery, + uid string, + key types.KeyHolder, +) map[string]types.DeviceKeysQuery { + // preset for complicated nested map struct + if _, ok := deviceKeysQueryMap[key.DeviceID]; !ok { + // make consistency + deviceKeysQueryMap[key.DeviceID] = types.DeviceKeysQuery{} + } + if deviceKeysQueryMap[key.DeviceID].Signature == nil { + mid := make(map[string]map[string]string) + midmap := deviceKeysQueryMap[key.DeviceID] + midmap.Signature = mid + deviceKeysQueryMap[key.DeviceID] = midmap + } + if deviceKeysQueryMap[key.DeviceID].Keys == nil { + mid := make(map[string]string) + midmap := deviceKeysQueryMap[key.DeviceID] + midmap.Keys = mid + deviceKeysQueryMap[key.DeviceID] = midmap + } + if _, ok := deviceKeysQueryMap[key.DeviceID].Signature[uid]; !ok { + // make consistency + deviceKeysQueryMap[key.DeviceID].Signature[uid] = make(map[string]string) + } + return deviceKeysQueryMap +} + +func bothKeyProcess( + ctx context.Context, + body *types.UploadEncryptSpecific, + userID, deviceID string, + database *storage.Database, + deviceKeys types.DeviceKeys, +) (err error) { + // insert one time keys firstly + onetimeKeys := body.OneTimeKey + for alKeyID, val := range onetimeKeys.KeyString { + al := (strings.Split(alKeyID, ":"))[0] + keyID := (strings.Split(alKeyID, ":"))[1] + keyInfo := val + keyStringTyp := ONETIMEKEYSTR + sig := "" + err = database.InsertKey(ctx, deviceID, userID, keyID, keyStringTyp, keyInfo, al, sig) + if err != nil { + return + } + } + for alKeyID, val := range onetimeKeys.KeyObject { + al := (strings.Split(alKeyID, ":"))[0] + keyID := (strings.Split(alKeyID, ":"))[1] + keyInfo := val.Key + keyObjectTyp := ONETIMEKEYSTR + sig := val.Signature[userID][fmt.Sprintf("%s:%s", "ed25519", deviceID)] + err = database.InsertKey(ctx, deviceID, userID, keyID, keyObjectTyp, keyInfo, al, sig) + if err != nil { + return + } + } + // insert device keys + keys := deviceKeys.Keys + sigs := deviceKeys.Signature + for alDevice, key := range keys { + al := (strings.Split(alDevice, ":"))[0] + keyTyp := DEVICEKEYSTR + keyInfo := key + keyID := "" + sig := sigs[userID][fmt.Sprintf("%s:%s", "ed25519", deviceID)] + err = database.InsertKey( + ctx, deviceID, userID, keyID, keyTyp, keyInfo, al, sig) + if err != nil { + return + } + } + return +} + +func dkeyProcess( + ctx context.Context, + userID, deviceID string, + database *storage.Database, + deviceKeys types.DeviceKeys, +) (err error) { + keys := deviceKeys.Keys + sigs := deviceKeys.Signature + for alDevice, key := range keys { + al := (strings.Split(alDevice, ":"))[0] + keyTyp := DEVICEKEYSTR + keyInfo := key + keyID := "" + sig := sigs[userID][fmt.Sprintf("%s:%s", "ed25519", deviceID)] + err = database.InsertKey(ctx, deviceID, userID, keyID, keyTyp, keyInfo, al, sig) + } + return +} + +func otmKeyProcess( + ctx context.Context, + body *types.UploadEncryptSpecific, + userID, deviceID string, + database *storage.Database, +) (err error) { + onetimeKeys := body.OneTimeKey + for alKeyID, val := range onetimeKeys.KeyString { + al := (strings.Split(alKeyID, ":"))[0] + keyID := (strings.Split(alKeyID, ":"))[1] + keyInfo := val + oneTimeKeyStringTyp := ONETIMEKEYSTR + sig := "" + err = database.InsertKey(ctx, deviceID, userID, keyID, oneTimeKeyStringTyp, keyInfo, al, sig) + if err != nil { + return + } + } + for alKeyID, val := range onetimeKeys.KeyObject { + al := (strings.Split(alKeyID, ":"))[0] + keyID := (strings.Split(alKeyID, ":"))[1] + keyInfo := val.Key + oneTimeKeyObjectTyp := ONETIMEKEYSTR + sig := val.Signature[userID][fmt.Sprintf("%s:%s", "ed25519", deviceID)] + err = database.InsertKey(ctx, deviceID, userID, keyID, oneTimeKeyObjectTyp, keyInfo, al, sig) + if err != nil { + return + } + } + return +} diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go b/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go index a8a5eb592..4ec1b184d 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go @@ -19,26 +19,21 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/encryptoapi/storage" "github.com/matrix-org/util" ) -const pathPrefixR0 = "/_matrix/client/r0" const pathPrefixUnstable = "/_matrix/client/unstable" +// Setup works for setting up encryption api server func Setup( apiMux *mux.Router, - cfg config.Dendrite, encryptionDB *storage.Database, - accountDB *accounts.Database, deviceDB *devices.Database, ) { - //r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() unstablemux := apiMux.PathPrefix(pathPrefixUnstable).Subrouter() unstablemux.Handle("/keys/upload/{deviceID}", @@ -56,16 +51,14 @@ func Setup( unstablemux.Handle("/keys/query", common.MakeAuthAPI("query keys", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { //vars := mux.Vars(req) - return QueryPKeys(req, encryptionDB, device.UserID, device.ID, deviceDB) + return QueryPKeys(req, encryptionDB, device.ID, deviceDB) }), ).Methods(http.MethodPost, http.MethodOptions) unstablemux.Handle("/keys/claim", common.MakeAuthAPI("claim keys", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - //vars := mux.Vars(req) - return ClaimOneTimeKeys(req, encryptionDB, device.UserID, device.ID, deviceDB) + return ClaimOneTimeKeys(req, encryptionDB) }), ).Methods(http.MethodPost, http.MethodOptions) - } diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_algorithm.go b/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_algorithm.go index d243028c6..0e230a9f9 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_algorithm.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_algorithm.go @@ -15,8 +15,8 @@ package storage import ( - "database/sql" "context" + "database/sql" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/encryptoapi/types" ) @@ -59,29 +59,29 @@ func (s *alStatements) prepare(db *sql.DB) (err error) { } // persist algorithms -func (ks *alStatements) insertAl( +func (s *alStatements) insertAl( ctx context.Context, txn *sql.Tx, userID, deviceID, algorithms string, ) error { - stmt := common.TxStmt(txn, ks.insertAlStmt) + stmt := common.TxStmt(txn, s.insertAlStmt) _, err := stmt.ExecContext(ctx, deviceID, userID, algorithms) return err } // select algorithms -func (ks *alStatements) selectAl( +func (s *alStatements) selectAl( ctx context.Context, txn *sql.Tx, - userID, deviceID string, + userID, deviceID string, ) (holder types.AlHolder, err error) { - stmt := common.TxStmt(txn, ks.selectAlStmt) + stmt := common.TxStmt(txn, s.selectAlStmt) row := stmt.QueryRowContext(ctx, userID, deviceID) single := types.AlHolder{} err = row.Scan( - &single.User_id, - &single.Device_id, - &single.Supported_algorithm, + &single.UserID, + &single.DeviceID, + &single.SupportedAlgorithm, ) return single, err } diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go b/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go index c3c7ed469..af30301ff 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go @@ -15,11 +15,11 @@ package storage import ( - "database/sql" "context" + "database/sql" + "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/encryptoapi/types" - "github.com/lib/pq" ) const keysSchema = ` @@ -96,104 +96,106 @@ func (s *keyStatements) prepare(db *sql.DB) (err error) { } // insert keys -func (ks *keyStatements) insertKey( +func (s *keyStatements) insertKey( ctx context.Context, txn *sql.Tx, deviceID, userID, keyID, keyTyp, keyInfo, algorithm, signature string, ) error { - stmt := common.TxStmt(txn, ks.insertKeyStmt) + stmt := common.TxStmt(txn, s.insertKeyStmt) _, err := stmt.ExecContext(ctx, deviceID, userID, keyID, keyTyp, keyInfo, algorithm, signature) return err } // select by user and device -func (ks *keyStatements) selectKey( +func (s *keyStatements) selectKey( ctx context.Context, txn *sql.Tx, deviceID, userID string, -) (holders []types.KeyHolder, err error) { - stmt := common.TxStmt(txn, ks.selectKeyStmt) +) ([]types.KeyHolder, error) { + holders := []types.KeyHolder{} + stmt := common.TxStmt(txn, s.selectKeyStmt) rows, err := stmt.QueryContext(ctx, userID, deviceID) - defer rows.Close() if err != nil { return nil, err } for rows.Next() { single := &types.KeyHolder{} - if err := rows.Scan( - &single.User_id, - &single.Device_id, - &single.Key_id, - &single.Key_type, + if err = rows.Scan( + &single.UserID, + &single.DeviceID, + &single.KeyID, + &single.KeyType, &single.Key, - &single.Key_algorithm, + &single.KeyAlgorithm, &single.Signature, ); err != nil { return nil, err } holders = append(holders, *single) } + err = rows.Close() return holders, err } // select single one for claim usage -func (ks *keyStatements) selectSingleKey( +func (s *keyStatements) selectSingleKey( ctx context.Context, userID, deviceID, algorithm string, ) (holder types.KeyHolder, err error) { - stmt := ks.selectSingleKeyStmt + stmt := s.selectSingleKeyStmt row := stmt.QueryRowContext(ctx, userID, deviceID, algorithm) if err != nil { return holder, err } - if err := row.Scan( - &holder.User_id, - &holder.Device_id, - &holder.Key_id, - &holder.Key_type, + if err = row.Scan( + &holder.UserID, + &holder.DeviceID, + &holder.KeyID, + &holder.KeyType, &holder.Key, - &holder.Key_algorithm, + &holder.KeyAlgorithm, &holder.Signature, ); err != nil { - deleteStmt := ks.deleteSingleKeyStmt - deleteStmt.ExecContext(ctx, userID, deviceID, algorithm, holder.Key_id) + deleteStmt := s.deleteSingleKeyStmt + _, err = deleteStmt.ExecContext(ctx, userID, deviceID, algorithm, holder.KeyID) return holder, err } return holder, err } // select details by given an array of devices -func (ks *keyStatements) selectInKeys( +func (s *keyStatements) selectInKeys( ctx context.Context, userID string, arr []string, ) (holders []types.KeyHolder, err error) { - rows := &sql.Rows{} - defer rows.Close() - stmt := ks.selectAllKeyStmt + rows := sql.Rows{} + rowsP := &rows + stmt := s.selectAllKeyStmt if len(arr) == 0 { - rows, err = stmt.QueryContext(ctx, userID, "device_key") + rowsP, err = stmt.QueryContext(ctx, userID, "device_key") } else { - stmt = ks.selectInKeysStmt + stmt = s.selectInKeysStmt list := pq.Array(arr) - rows, err = stmt.QueryContext(ctx, userID, list) + rowsP, err = stmt.QueryContext(ctx, userID, list) } if err != nil { return nil, err } for rows.Next() { single := &types.KeyHolder{} - if err := rows.Scan( - &single.User_id, - &single.Device_id, - &single.Key_id, - &single.Key_type, + if err = rows.Scan( + &single.UserID, + &single.DeviceID, + &single.KeyID, + &single.KeyType, &single.Key, - &single.Key_algorithm, + &single.KeyAlgorithm, &single.Signature, ); err != nil { return nil, err } holders = append(holders, *single) } + err = rowsP.Close() return holders, err } diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/storage/storage.go b/src/github.com/matrix-org/dendrite/encryptoapi/storage/storage.go index 40e4c1fe2..2ed567a7d 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/storage/storage.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/storage/storage.go @@ -15,9 +15,9 @@ package storage import ( + "context" "database/sql" "github.com/matrix-org/dendrite/common" - "context" "github.com/matrix-org/dendrite/encryptoapi/types" "strings" ) @@ -47,7 +47,7 @@ func NewDatabase(dataSourceName string) (*Database, error) { return &Database{db: db, keyStatements: keyStatement, alStatements: alStatement}, nil } -// insert device key +// InsertKey insert device key func (d *Database) InsertKey( ctx context.Context, deviceID, userID, keyID, keyTyp, keyInfo, al, sig string, @@ -58,7 +58,7 @@ func (d *Database) InsertKey( return } -// for key upload response usage a map from key algorithm to sum to counterpart +// SelectOneTimeKeyCount for key upload response usage a map from key algorithm to sum to counterpart func (d *Database) SelectOneTimeKeyCount( ctx context.Context, deviceID, userID string, @@ -67,11 +67,11 @@ func (d *Database) SelectOneTimeKeyCount( err = common.WithTransaction(d.db, func(txn *sql.Tx) error { elems, err := d.keyStatements.selectKey(ctx, txn, deviceID, userID) for _, val := range elems { - if _, ok := m[val.Key_algorithm]; !ok { - m[val.Key_algorithm] = 0 + if _, ok := m[val.KeyAlgorithm]; !ok { + m[val.KeyAlgorithm] = 0 } - if val.Key_type == "one_time_key" { - m[val.Key_algorithm] += 1 + if val.KeyType == "one_time_key" { + m[val.KeyAlgorithm]++ } } return err @@ -79,7 +79,7 @@ func (d *Database) SelectOneTimeKeyCount( return } -// query keys in a range of devices +// QueryInRange query keys in a range of devices func (d *Database) QueryInRange( ctx context.Context, userID string, @@ -89,30 +89,30 @@ func (d *Database) QueryInRange( return } -// persist algorithms +// InsertAl persist algorithms func (d *Database) InsertAl( ctx context.Context, uid, device string, al []string, ) (err error) { err = common.WithTransaction(d.db, func(txn *sql.Tx) (err error) { - d.alStatements.insertAl(ctx, txn, uid, device, strings.Join(al, ",")) + err = d.alStatements.insertAl(ctx, txn, uid, device, strings.Join(al, ",")) return }) return } -// select algorithms +// SelectAl select algorithms func (d *Database) SelectAl( ctx context.Context, uid, device string, ) (res []string, err error) { err = common.WithTransaction(d.db, func(txn *sql.Tx) (err error) { holder, err := d.alStatements.selectAl(ctx, txn, uid, device) - res = strings.Split(holder.Supported_algorithm, ",") + res = strings.Split(holder.SupportedAlgorithm, ",") return }) return } -// claim for one time key one for once +// SelectOneTimeKeySingle claim for one time key one for once func (d *Database) SelectOneTimeKeySingle( ctx context.Context, userID, deviceID, algorithm string, diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/types/claim.go b/src/github.com/matrix-org/dendrite/encryptoapi/types/claim.go index 80a786d20..858696f1e 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/types/claim.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/types/claim.go @@ -14,41 +14,14 @@ package types -/* - { - "timeout": 10000, - "one_time_keys": { - "@alice:example.com": { - "JLAFKJWSCS": "curve25519" - } - } - } -*/ +// ClaimRequest structure type ClaimRequest struct { Timeout int64 `json:"timeout"` ClaimDetail map[string]map[string]string `json:"one_time_keys"` } -/* - { - "failures": {}, - "one_time_keys": { - "@alice:example.com": { - "JLAFKJWSCS": { - "signed_curve25519:AAAAHg": { - "key": "zKbLg+NrIjpnagy+pIY6uPL4ZwEG2v+8F9lmgsnlZzs", - "signatures": { - "@alice:example.com": { - "ed25519:JLAFKJWSCS": "FLWxXqGbwrb8SM3Y795eB6OA8bwBcoMZFXBqnTn58AYWZSqiD45tlBVcDa2L7RwdKXebW/VzDlnfVJ+9jok1Bw" - } - } - } - } - } - } - } -*/ +// ClaimResponse structure type ClaimResponse struct { - Failures map[string]interface{} `json:"failures"` + Failures map[string]interface{} `json:"failures"` ClaimBody map[string]map[string]map[string]interface{} `json:"one_time_keys"` } diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/types/query.go b/src/github.com/matrix-org/dendrite/encryptoapi/types/query.go index 1c5dba3a2..c7dae9ea7 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/types/query.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/types/query.go @@ -14,63 +14,30 @@ package types -/* - { - "timeout": 10000, - "device_keys": { - "@alice:example.com": ["DISYYYX","XYIISONM"] - } - } -*/ +// QueryRequest structure type QueryRequest struct { - Timeout int64 `json:"timeout"` + Timeout int64 `json:"timeout"` DeviceKeys map[string]interface{} `json:"device_keys"` - Token string `json:"token"` + Token string `json:"token"` } -/* - { - "failures": {}, - "device_keys": { - "@alice:example.com": { - "JLAFKJWSCS": { - "user_id": "@alice:example.com", - "device_id": "JLAFKJWSCS", - "algorithms": [ - "m.olm.curve25519-aes-sha256", - "m.megolm.v1.aes-sha" - ], - "keys": { - "curve25519:JLAFKJWSCS": "3C5BFWi2Y8MaVvjM8M22DBmh24PmgR0nPvJOIArzgyI", - "ed25519:JLAFKJWSCS": "lEuiRJBit0IG6nUf5pUzWTUEsRVVe/HJkoKuEww9ULI" - }, - "signatures": { - "@alice:example.com": { - "ed25519:JLAFKJWSCS": "dSO80A01XiigH3uBiDVx/EjzaoycHcjq9lfQX0uWsqxl2giMIiSPR8a4d291W1ihKJL/a+myXS367WT6NAIcBA" - } - }, - "unsigned": { - "device_display_name": "Alice's mobile phone" - } - } - } - } - } -*/ +// QueryResponse structure type QueryResponse struct { Failure map[string]interface{} `json:"failures"` DeviceKeys map[string]map[string]DeviceKeysQuery `json:"device_keys"` } +// DeviceKeysQuery structure type DeviceKeysQuery struct { - UserId string `json:"user_id"` - DeviceId string `json:"device_id"` + UserID string `json:"user_id"` + DeviceID string `json:"device_id"` Algorithm []string `json:"algorithms"` Keys map[string]string `json:"keys"` Signature map[string]map[string]string `json:"signatures"` Unsigned UnsignedDeviceInfo `json:"unsigned"` } +// UnsignedDeviceInfo structure type UnsignedDeviceInfo struct { Info string `json:"device_display_name"` } diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/types/storage.go b/src/github.com/matrix-org/dendrite/encryptoapi/types/storage.go index 55815420a..c3d098398 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/types/storage.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/types/storage.go @@ -14,17 +14,20 @@ package types +// KeyHolder structure type KeyHolder struct { - User_id, - Device_id, - Signature, key, - Key_algorithm, - Key_id, + UserID, + DeviceID, + Signature, + KeyAlgorithm, + KeyID, Key, - Key_type string + KeyType string } + +// AlHolder structure type AlHolder struct { - User_id, - Device_id, - Supported_algorithm string + UserID, + DeviceID, + SupportedAlgorithm string } diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/types/upload.go b/src/github.com/matrix-org/dendrite/encryptoapi/types/upload.go index b2f542075..7e8a62c13 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/types/upload.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/types/upload.go @@ -14,135 +14,46 @@ package types -/* - { - "device_keys": { - "user_id": "@alice:example.com", - "device_id": "JLAFKJWSCS", - "algorithms": [ - "m.olm.curve25519-aes-sha256", - "m.megolm.v1.aes-sha" - ], - "keys": { - "curve25519:JLAFKJWSCS": "3C5BFWi2Y8MaVvjM8M22DBmh24PmgR0nPvJOIArzgyI", - "ed25519:JLAFKJWSCS": "lEuiRJBit0IG6nUf5pUzWTUEsRVVe/HJkoKuEww9ULI" - }, - "signatures": { - "@alice:example.com": { - "ed25519:JLAFKJWSCS": "dSO80A01XiigH3uBiDVx/EjzaoycHcjq9lfQX0uWsqxl2giMIiSPR8a4d291W1ihKJL/a+myXS367WT6NAIcBA" - } - } - }, - "one_time_keys": { - "curve25519:AAAAAQ": "/qyvZvwjiTxGdGU0RCguDCLeR+nmsb3FfNG3/Ve4vU8", - "signed_curve25519:AAAAHg": { - "key": "zKbLg+NrIjpnagy+pIY6uPL4ZwEG2v+8F9lmgsnlZzs", - "signatures": { - "@alice:example.com": { - "ed25519:JLAFKJWSCS": "FLWxXqGbwrb8SM3Y795eB6OA8bwBcoMZFXBqnTn58AYWZSqiD45tlBVcDa2L7RwdKXebW/VzDlnfVJ+9jok1Bw" - } - } - }, - "signed_curve25519:AAAAHQ": { - "key": "j3fR3HemM16M7CWhoI4Sk5ZsdmdfQHsKL1xuSft6MSw", - "signatures": { - "@alice:example.com": { - "ed25519:JLAFKJWSCS": "IQeCEPb9HFk217cU9kw9EOiusC6kMIkoIRnbnfOh5Oc63S1ghgyjShBGpu34blQomoalCyXWyhaaT3MrLZYQAA" - } - } - } - } - } -*/ +// UploadEncrypt structure type UploadEncrypt struct { DeviceKeys DeviceKeys `json:"device_keys"` OneTimeKey map[string]interface{} `json:"one_time_keys"` } + +// UploadEncryptSpecific structure type UploadEncryptSpecific struct { DeviceKeys DeviceKeys `json:"device_keys"` OneTimeKey OneTimeKeySpecific `json:"one_time_keys"` } -/* - { - "one_time_key_counts": { - "curve25519": 10, - "signed_curve25519": 20 - } - } -*/ +// UploadResponse structure type UploadResponse struct { Count map[string]int `json:"one_time_key_counts"` } -/* - "device_keys": { - "user_id": "@alice:example.com", - "device_id": "JLAFKJWSCS", - "algorithms": [ - "m.olm.curve25519-aes-sha256", - "m.megolm.v1.aes-sha" - ], - "keys": { - "curve25519:JLAFKJWSCS": "3C5BFWi2Y8MaVvjM8M22DBmh24PmgR0nPvJOIArzgyI", - "ed25519:JLAFKJWSCS": "lEuiRJBit0IG6nUf5pUzWTUEsRVVe/HJkoKuEww9ULI" - }, - "signatures": { - "@alice:example.com": { - "ed25519:JLAFKJWSCS": "dSO80A01XiigH3uBiDVx/EjzaoycHcjq9lfQX0uWsqxl2giMIiSPR8a4d291W1ihKJL/a+myXS367WT6NAIcBA" - } - } - } -*/ +// DeviceKeys structure type DeviceKeys struct { - UserId string `json:"user_id"` - DeviceId string `json:"device_id"` + UserID string `json:"user_id"` + DeviceID string `json:"device_id"` Algorithm []string `json:"algorithms"` Keys map[string]string `json:"keys"` Signature map[string]map[string]string `json:"signatures"` } -/* - "signed_curve25519:AAAAHg": { - "key": "zKbLg+NrIjpnagy+pIY6uPL4ZwEG2v+8F9lmgsnlZzs", - "signatures": { - "@alice:example.com": { - "ed25519:JLAFKJWSCS": "FLWxXqGbwrb8SM3Y795eB6OA8bwBcoMZFXBqnTn58AYWZSqiD45tlBVcDa2L7RwdKXebW/VzDlnfVJ+9jok1Bw" - } - } - } -*/ +// KeyObject structure type KeyObject struct { Key string `json:"key"` Signature map[string]map[string]string `json:"signatures"` } -/* - "one_time_keys": { - "curve25519:AAAAAQ": "/qyvZvwjiTxGdGU0RCguDCLeR+nmsb3FfNG3/Ve4vU8", - "signed_curve25519:AAAAHg": { - "key": "zKbLg+NrIjpnagy+pIY6uPL4ZwEG2v+8F9lmgsnlZzs", - "signatures": { - "@alice:example.com": { - "ed25519:JLAFKJWSCS": "FLWxXqGbwrb8SM3Y795eB6OA8bwBcoMZFXBqnTn58AYWZSqiD45tlBVcDa2L7RwdKXebW/VzDlnfVJ+9jok1Bw" - } - } - }, - "signed_curve25519:AAAAHQ": { - "key": "j3fR3HemM16M7CWhoI4Sk5ZsdmdfQHsKL1xuSft6MSw", - "signatures": { - "@alice:example.com": { - "ed25519:JLAFKJWSCS": "IQeCEPb9HFk217cU9kw9EOiusC6kMIkoIRnbnfOh5Oc63S1ghgyjShBGpu34blQomoalCyXWyhaaT3MrLZYQAA" - } - } - } - } -*/ +// OneTimeKey structure type OneTimeKey struct { //KeyString map[string]string //KeyObject map[string]KeyObject KeySth map[string]interface{} } + +// OneTimeKeySpecific structure type OneTimeKeySpecific struct { KeyString map[string]string KeyObject map[string]KeyObject From 77706648784632d9085fde7bf177117c6b2eac80 Mon Sep 17 00:00:00 2001 From: terrill <314156936@qq.com> Date: Mon, 16 Jul 2018 18:54:19 +0800 Subject: [PATCH 09/18] bug fix at testing response val inconsistency --- .../matrix-org/dendrite/encryptoapi/routing/keys.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/routing/keys.go b/src/github.com/matrix-org/dendrite/encryptoapi/routing/keys.go index 60a8bfa41..66439316c 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/routing/keys.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/routing/keys.go @@ -161,12 +161,12 @@ func QueryPKeys( deviceKeysQueryMap = presetDeviceKeysQueryMap(deviceKeysQueryMap, uid, key) // load for accomplishment single := deviceKeysQueryMap[key.DeviceID] - resKey := fmt.Sprintf("@%s:%s", key.KeyAlgorithm, key.DeviceID) + resKey := fmt.Sprintf("%s:%s", key.KeyAlgorithm, key.DeviceID) resBody := key.Key single.Keys[resKey] = resBody single.DeviceID = key.DeviceID single.UserID = key.UserID - single.Signature[uid][fmt.Sprintf("@%s:%s", "ed25519", key.DeviceID)] = key.Signature + single.Signature[uid][fmt.Sprintf("%s:%s", "ed25519", key.DeviceID)] = key.Signature single.Algorithm, err = takeAL(req.Context(), *encryptionDB, key.UserID, key.DeviceID) localpart, _, _ := gomatrixserverlib.SplitID('@', uid) device, _ := deviceDB.GetDeviceByID(req.Context(), localpart, deviceID) From 57e6eb73dc457b9839b4c42300254c567a936b49 Mon Sep 17 00:00:00 2001 From: terrill <314156936@qq.com> Date: Wed, 18 Jul 2018 10:53:27 +0800 Subject: [PATCH 10/18] send to device (STD) done, cooperate with key centre encryptoapi --- .../auth/storage/devices/devices_table.go | 6 +- .../encryptoapi/storage/encrypt_keys_table.go | 39 +++-- .../dendrite/syncapi/routing/routing.go | 13 +- .../dendrite/syncapi/routing/std.go | 89 ++++++++++ .../syncapi/storage/send_to_device_table.go | 162 ++++++++++++++++++ .../dendrite/syncapi/storage/syncserver.go | 106 +++++++++++- .../dendrite/syncapi/sync/requestpool.go | 4 + .../matrix-org/dendrite/syncapi/syncapi.go | 2 +- .../dendrite/syncapi/types/types.go | 29 +++- 9 files changed, 433 insertions(+), 17 deletions(-) create mode 100644 src/github.com/matrix-org/dendrite/syncapi/routing/std.go create mode 100644 src/github.com/matrix-org/dendrite/syncapi/storage/send_to_device_table.go diff --git a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/devices_table.go b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/devices_table.go index 96d6521d8..e5be77a62 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/devices_table.go +++ b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/devices_table.go @@ -60,7 +60,9 @@ const selectDeviceByIDSQL = "" + "SELECT display_name FROM device_devices WHERE localpart = $1 and device_id = $2" const selectDevicesByLocalpartSQL = "" + - "SELECT device_id, display_name FROM device_devices WHERE localpart = $1" + // todo : display name still has a problem when value is null + //"SELECT device_id, display_name FROM device_devices WHERE localpart = $1" + "SELECT device_id FROM device_devices WHERE localpart = $1" const updateDeviceNameSQL = "" + "UPDATE device_devices SET display_name = $1 WHERE localpart = $2 AND device_id = $3" @@ -197,6 +199,8 @@ func (s *devicesStatements) selectDevicesByLocalpart( for rows.Next() { var dev authtypes.Device err = rows.Scan(&dev.ID) + // todo: display name still has a problem when value is null + //err = rows.Scan(&dev.ID, &dev.DisplayName) if err != nil { return devices, err } diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go b/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go index af30301ff..8c822fbd2 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go @@ -167,20 +167,37 @@ func (s *keyStatements) selectInKeys( ctx context.Context, userID string, arr []string, -) (holders []types.KeyHolder, err error) { - rows := sql.Rows{} - rowsP := &rows +) ([]types.KeyHolder, error) { + holders := []types.KeyHolder{} stmt := s.selectAllKeyStmt if len(arr) == 0 { - rowsP, err = stmt.QueryContext(ctx, userID, "device_key") - } else { - stmt = s.selectInKeysStmt - list := pq.Array(arr) - rowsP, err = stmt.QueryContext(ctx, userID, list) + // mapping for all device keys + rowsP, err := stmt.QueryContext(ctx, userID, "device_key") + if err != nil { + return nil, err + } + holders, err = injectKeyHolder(rowsP, holders) + if err != nil { + return nil, err + } + err = rowsP.Close() + return holders, err } + stmt = s.selectInKeysStmt + list := pq.Array(arr) + rowsP, err := stmt.QueryContext(ctx, userID, list) if err != nil { return nil, err } + holders, err = injectKeyHolder(rowsP, holders) + if err != nil { + return nil, err + } + err = rowsP.Close() + return holders, err +} + +func injectKeyHolder(rows *sql.Rows, keyHolder []types.KeyHolder) (holders []types.KeyHolder, err error) { for rows.Next() { single := &types.KeyHolder{} if err = rows.Scan( @@ -194,8 +211,8 @@ func (s *keyStatements) selectInKeys( ); err != nil { return nil, err } - holders = append(holders, *single) + keyHolder = append(keyHolder, *single) } - err = rowsP.Close() - return holders, err + holders = keyHolder + return } diff --git a/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go b/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go index e5a906b08..f4e6fa959 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go @@ -27,10 +27,12 @@ import ( ) const pathPrefixR0 = "/_matrix/client/r0" +const pathPrefixUnstable = "/_matrix/client/unstable" // Setup configures the given mux with sync-server listeners -func Setup(apiMux *mux.Router, srp *sync.RequestPool, syncDB *storage.SyncServerDatabase, deviceDB *devices.Database) { +func Setup(apiMux *mux.Router, srp *sync.RequestPool, syncDB *storage.SyncServerDatabase, deviceDB *devices.Database, notifier *sync.Notifier) { r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() + unstablemux := apiMux.PathPrefix(pathPrefixUnstable).Subrouter() r0mux.Handle("/sync", common.MakeAuthAPI("sync", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return srp.OnIncomingSyncRequest(req, device) @@ -50,4 +52,13 @@ func Setup(apiMux *mux.Router, srp *sync.RequestPool, syncDB *storage.SyncServer vars := mux.Vars(req) return OnIncomingStateTypeRequest(req, syncDB, vars["roomID"], vars["type"], vars["stateKey"]) })).Methods(http.MethodGet, http.MethodOptions) + + unstablemux.Handle("/sendToDevice/{eventType}/{txnId}", + common.MakeAuthAPI("look up changes", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars := mux.Vars(req) + eventType := vars["eventType"] + txnID := vars["txnId"] + return SendToDevice(req, device.UserID, syncDB, deviceDB, eventType, txnID, notifier) + }), + ).Methods(http.MethodPut, http.MethodOptions) } diff --git a/src/github.com/matrix-org/dendrite/syncapi/routing/std.go b/src/github.com/matrix-org/dendrite/syncapi/routing/std.go new file mode 100644 index 000000000..2c35471de --- /dev/null +++ b/src/github.com/matrix-org/dendrite/syncapi/routing/std.go @@ -0,0 +1,89 @@ +package routing + +import ( + "encoding/json" + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/sync" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "net/http" +) + +// SendToDevice this is a function for calling process of send-to-device messages those bypassed DAG +func SendToDevice( + req *http.Request, + sender string, + syncDB *storage.SyncServerDatabase, + deviceDB *devices.Database, + eventType, txnID string, + notifier *sync.Notifier, +) util.JSONResponse { + ctx := req.Context() + stdRq := types.StdRequest{} + httputil.UnmarshalJSONRequest(req, &stdRq) + for uid, deviceMap := range stdRq.Sender { + + // federation consideration todo: + // if uid is remote domain a fed process should go + if false { + // federation process + return util.JSONResponse{} + } + + // uid is local domain + for device, cont := range deviceMap { + jsonBuffer, err := json.Marshal(cont) + if err != nil { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: struct{}{}, + } + } + ev := types.StdHolder{ + Sender: sender, + Event: jsonBuffer, + EventTyp: eventType, + } + var pos int64 + + // wildcard all devices + if device == "*" { + var deviceCollection []authtypes.Device + var localpart string + localpart, _, _ = gomatrixserverlib.SplitID('@', uid) + deviceCollection, err = deviceDB.GetDevicesByLocalpart(ctx, localpart) + for _, val := range deviceCollection { + pos, err = syncDB.InsertStdMessage(ctx, ev, txnID, uid, val.ID) + notifier.OnNewEvent(nil, uid, types.StreamPosition(pos)) + } + if err != nil { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: struct{}{}, + } + } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } + } + pos, err = syncDB.InsertStdMessage(ctx, ev, txnID, uid, device) + if err != nil { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: struct{}{}, + } + } + notifier.OnNewEvent(nil, uid, types.StreamPosition(pos)) + } + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } +} diff --git a/src/github.com/matrix-org/dendrite/syncapi/storage/send_to_device_table.go b/src/github.com/matrix-org/dendrite/syncapi/storage/send_to_device_table.go new file mode 100644 index 000000000..33d1022cf --- /dev/null +++ b/src/github.com/matrix-org/dendrite/syncapi/storage/send_to_device_table.go @@ -0,0 +1,162 @@ +package storage + +import ( + "context" + "database/sql" + + "github.com/lib/pq" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/syncapi/types" +) + +// we treat send to device as abbrev as STD in the context below. + +const sendToDeviceSchema = ` +CREATE TABLE IF NOT EXISTS syncapi_send_to_device ( + id BIGINT PRIMARY KEY DEFAULT nextval('syncapi_stream_id'), + txn_id TEXT NOT NULL, + sender TEXT NOT NULL, + event_type TEXT NOT NULL, + target_device_id TEXT NOT NULL, + target_user_id TEXT NOT NULL, + event_json TEXT NOT NULL, + del_read INTEGER DEFAULT 0, + max_read BIGINT DEFAULT currval('syncapi_stream_id') , +CONSTRAINT syncapi_send_to_device_unique UNIQUE (txn_id, target_device_id, target_user_id) +); +` + +const insertSTDSQL = "" + + "INSERT INTO syncapi_send_to_device (" + + " sender, event_type, target_user_id, target_device_id, txn_id, event_json" + + ") VALUES ($1, $2, $3, $4, $5, $6) RETURNING id" + +const deleteSTDSQL = "" + + "DELETE FROM syncapi_send_to_device WHERE target_user_id = $1 AND target_device_id = $2 AND max_read < $3 AND del_read = 1" + +const selectSTDEventsInRangeSQL = "" + + "SELECT id, sender, event_type, event_json FROM syncapi_send_to_device" + + " WHERE target_user_id = $1 AND target_device_id = $2 AND id <= $3" + + " ORDER BY id LIMIT 100 " + +const updateSTDEventSQL = "" + + "UPDATE syncapi_send_to_device SET del_read = 1 , max_read = $1 WHERE id = ANY($2)" + +const selectMaxSTDIDSQL = "" + + "SELECT MAX(id) FROM syncapi_send_to_device" + +type stdEventsStatements struct { + insertStdEventStmt *sql.Stmt + selectStdEventsInRangeStmt *sql.Stmt + deleteStdEventStmt *sql.Stmt + selectStdIDStmt *sql.Stmt + updateStdStmt *sql.Stmt +} + +func (s *stdEventsStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(sendToDeviceSchema) + if err != nil { + return + } + if s.insertStdEventStmt, err = db.Prepare(insertSTDSQL); err != nil { + return + } + if s.selectStdEventsInRangeStmt, err = db.Prepare(selectSTDEventsInRangeSQL); err != nil { + return + } + if s.deleteStdEventStmt, err = db.Prepare(deleteSTDSQL); err != nil { + return + } + if s.selectStdIDStmt, err = db.Prepare(selectMaxSTDIDSQL); err != nil { + return + } + if s.updateStdStmt, err = db.Prepare(updateSTDEventSQL); err != nil { + return + } + return +} + +func (s *stdEventsStatements) insertStdEvent( + ctx context.Context, stdEvent types.StdHolder, + transactionID string, targetUID, targetDevice string, +) (streamPos int64, err error) { + err = s.insertStdEventStmt.QueryRowContext( + ctx, + stdEvent.Sender, + stdEvent.EventTyp, + targetUID, + targetDevice, + transactionID, + stdEvent.Event, + ).Scan(&streamPos) + return +} + +func (s *stdEventsStatements) deleteStdEvent( + ctx context.Context, userID, deviceID string, + idUpBound int64, +) error { + _, err := s.deleteStdEventStmt.ExecContext(ctx, userID, deviceID, idUpBound) + return err +} + +func (s *stdEventsStatements) selectStdEventsInRange( + ctx context.Context, txn *sql.Tx, + targetUserID, targetDeviceID string, + endPos int64, +) ([]types.StdHolder, error) { + stdHolder := []types.StdHolder{} + stmt := common.TxStmt(txn, s.selectStdEventsInRangeStmt) + rows, err := stmt.QueryContext(ctx, targetUserID, targetDeviceID, endPos) + if err != nil { + return nil, err + } + for rows.Next() { + holder := types.StdHolder{} + var ( + id int64 + sender string + eventType string + eventJSON []byte + ) + if err = rows.Scan(&id, &sender, &eventType, &eventJSON); err != nil { + closeErr := rows.Close() + if closeErr != nil { + return nil, closeErr + } + return nil, err + } + holder.StreamID = id + holder.Sender = sender + holder.Event = eventJSON + holder.EventTyp = eventType + stdHolder = append(stdHolder, holder) + } + err = rows.Close() + if err != nil { + return nil, err + } + // update events with read mark + update := []int64{} + for _, val := range stdHolder { + update = append(update, val.StreamID) + } + updateStmt := common.TxStmt(txn, s.updateStdStmt) + _, err = updateStmt.ExecContext(ctx, endPos, pq.Array(update)) + if err != nil { + return nil, err + } + return stdHolder, nil +} + +func (s *stdEventsStatements) selectMaxStdID( + ctx context.Context, txn *sql.Tx, +) (id int64, err error) { + var nullableID sql.NullInt64 + stmt := common.TxStmt(txn, s.selectStdIDStmt) + err = stmt.QueryRowContext(ctx).Scan(&nullableID) + if nullableID.Valid { + id = nullableID.Int64 + } + return +} diff --git a/src/github.com/matrix-org/dendrite/syncapi/storage/syncserver.go b/src/github.com/matrix-org/dendrite/syncapi/storage/syncserver.go index 84417a348..d607005c9 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/storage/syncserver.go +++ b/src/github.com/matrix-org/dendrite/syncapi/storage/syncserver.go @@ -23,8 +23,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/roomserver/api" - // Import the postgres database driver. - _ "github.com/lib/pq" + + "encoding/json" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -54,6 +54,7 @@ type SyncServerDatabase struct { events outputRoomEventsStatements roomstate currentRoomStateStatements invites inviteEventsStatements + stdMsg stdEventsStatements } // NewSyncServerDatabase creates a new sync server database @@ -78,6 +79,9 @@ func NewSyncServerDatabase(dataSourceName string) (*SyncServerDatabase, error) { if err := d.invites.prepare(d.db); err != nil { return nil, err } + if err := d.stdMsg.prepare(d.db); err != nil { + return nil, err + } return &d, nil } @@ -212,6 +216,13 @@ func (d *SyncServerDatabase) syncStreamPositionTx( if maxInviteID > maxID { maxID = maxInviteID } + maxStdID, err := d.stdMsg.selectMaxStdID(ctx, txn) + if err != nil { + return 0, err + } + if maxStdID > maxID { + maxID = maxStdID + } return types.StreamPosition(maxID), nil } @@ -678,3 +689,94 @@ func getMembershipFromEvent(ev *gomatrixserverlib.Event, userID string) string { } return "" } + +/* +send to device messaging implementation +del / maxID / select in range / insert +*/ + +// DelStdMessage delete message for a given maxID, those below would be deleted +func (d *SyncServerDatabase) DelStdMessage( + ctx context.Context, targetUID, targetDevice string, maxID int64, +) (err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + err := d.stdMsg.deleteStdEvent(ctx, targetUID, targetDevice, maxID) + return err + }) + return +} + +// InsertStdMessage insert std message +func (d *SyncServerDatabase) InsertStdMessage( + ctx context.Context, stdEvent types.StdHolder, transactionID, targetUID, targetDevice string, +) (pos int64, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + curPos, err := d.stdMsg.insertStdEvent(ctx, stdEvent, transactionID, targetUID, targetDevice) + pos = curPos + return err + }) + return +} + +// SelectMaxStdID select maximum id in std stream +func (d *SyncServerDatabase) SelectMaxStdID( + ctx context.Context, +) (maxID int64, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + max, err := d.stdMsg.selectMaxStdID(ctx, txn) + maxID = max + return err + }) + return +} + +// SelectRangedStd select a range of std messages +func (d *SyncServerDatabase) SelectRangedStd( + ctx context.Context, + targetUserID, targetDeviceID string, + endPos int64, +) (holder []types.StdHolder, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + list, err := d.stdMsg.selectStdEventsInRange(ctx, txn, targetUserID, targetDeviceID, endPos) + holder = list + return err + }) + return +} + +// StdEXT : send to device extension process +func StdEXT( + ctx context.Context, + syncDB *SyncServerDatabase, + respIn types.Response, + userID, deviceID string, + since int64, +) (respOut *types.Response) { + respOut = &respIn + // when extension works at the very beginning + err := syncDB.stdMsg.deleteStdEvent(ctx, userID, deviceID, since) + if err != nil { + return + } + // when err is nil, these before res should be tagged omitted, + // when next /sync is coming , and err is nil , all those omitted. + res, err := syncDB.SelectRangedStd(ctx, userID, deviceID, since) + if err != nil { + return + } + //toDevice := &types.ToDevice{} + mid := []types.StdEvent{} + //toDevice.StdEvent = mid + for _, val := range res { + ev := types.StdEvent{} + ev.Sender = val.Sender + ev.Type = val.EventTyp + err := json.Unmarshal(val.Event, &ev.Content) + if err != nil { + return + } + mid = append(mid, ev) + } + respOut.ToDevice.StdEvent = mid + return +} diff --git a/src/github.com/matrix-org/dendrite/syncapi/sync/requestpool.go b/src/github.com/matrix-org/dendrite/syncapi/sync/requestpool.go index 5c560ff52..7092a09b3 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/sync/requestpool.go +++ b/src/github.com/matrix-org/dendrite/syncapi/sync/requestpool.go @@ -106,6 +106,10 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype // can respond syncData, err := rp.currentSyncForUser(*syncReq, currPos) + + // std extension consideration + syncData = storage.StdEXT(syncReq.ctx, rp.db, *syncData, syncReq.device.UserID, syncReq.device.ID, int64(currPos)) + if err != nil { return httputil.LogThenError(req, err) } diff --git a/src/github.com/matrix-org/dendrite/syncapi/syncapi.go b/src/github.com/matrix-org/dendrite/syncapi/syncapi.go index 2db54c3ce..c1de63c2b 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/syncapi.go +++ b/src/github.com/matrix-org/dendrite/syncapi/syncapi.go @@ -71,5 +71,5 @@ func SetupSyncAPIComponent( logrus.WithError(err).Panicf("failed to start client data consumer") } - routing.Setup(base.APIMux, requestPool, syncDB, deviceDB) + routing.Setup(base.APIMux, requestPool, syncDB, deviceDB, notifier) } diff --git a/src/github.com/matrix-org/dendrite/syncapi/types/types.go b/src/github.com/matrix-org/dendrite/syncapi/types/types.go index d0b1c38ab..e8f4e02a5 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/types/types.go +++ b/src/github.com/matrix-org/dendrite/syncapi/types/types.go @@ -50,6 +50,32 @@ type Response struct { Invite map[string]InviteResponse `json:"invite"` Leave map[string]LeaveResponse `json:"leave"` } `json:"rooms"` + ToDevice ToDevice `json:"to_device"` +} + +// StdHolder represents send to device response from db +type StdHolder struct { + StreamID int64 + Sender string + EventTyp string + Event []byte +} + +// StdRequest represents send to device request format +type StdRequest struct { + Sender map[string]map[string]interface{} `json:"messages"` +} + +// ToDevice represents a middleware for response send to device +type ToDevice struct { + StdEvent []StdEvent `json:"events"` +} + +// StdEvent represents send to device event format +type StdEvent struct { + Sender string `json:"sender"` + Type string `json:"type"` + Content interface{} `json:"content"` } // NewResponse creates an empty response with initialised maps. @@ -81,7 +107,8 @@ func (r *Response) IsEmpty() bool { len(r.Rooms.Invite) == 0 && len(r.Rooms.Leave) == 0 && len(r.AccountData.Events) == 0 && - len(r.Presence.Events) == 0 + len(r.Presence.Events) == 0 && + len(r.ToDevice.StdEvent) == 0 } // JoinResponse represents a /sync response for a room which is under the 'join' key. From d51521f4986d130ae8db4905401d363a50a9de99 Mon Sep 17 00:00:00 2001 From: terrill <314156936@qq.com> Date: Fri, 27 Jul 2018 10:07:06 +0800 Subject: [PATCH 11/18] unused value moved out --- .../matrix-org/dendrite/syncapi/routing/routing.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go b/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go index f4e6fa959..10d650a20 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go @@ -34,21 +34,21 @@ func Setup(apiMux *mux.Router, srp *sync.RequestPool, syncDB *storage.SyncServer r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() unstablemux := apiMux.PathPrefix(pathPrefixUnstable).Subrouter() - r0mux.Handle("/sync", common.MakeAuthAPI("sync", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + r0mux.Handle("/sync", common.MakeAuthAPI("sync", nil, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return srp.OnIncomingSyncRequest(req, device) })).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/state", common.MakeAuthAPI("room_state", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + r0mux.Handle("/rooms/{roomID}/state", common.MakeAuthAPI("room_state", nil, func(req *http.Request, device *authtypes.Device) util.JSONResponse { vars := mux.Vars(req) return OnIncomingStateRequest(req, syncDB, vars["roomID"]) })).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/state/{type}", common.MakeAuthAPI("room_state", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + r0mux.Handle("/rooms/{roomID}/state/{type}", common.MakeAuthAPI("room_state", nil, func(req *http.Request, device *authtypes.Device) util.JSONResponse { vars := mux.Vars(req) return OnIncomingStateTypeRequest(req, syncDB, vars["roomID"], vars["type"], "") })).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", common.MakeAuthAPI("room_state", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", common.MakeAuthAPI("room_state", nil, func(req *http.Request, device *authtypes.Device) util.JSONResponse { vars := mux.Vars(req) return OnIncomingStateTypeRequest(req, syncDB, vars["roomID"], vars["type"], vars["stateKey"]) })).Methods(http.MethodGet, http.MethodOptions) From 3ce007c3f8a7d1e54bbb203f5a284c459002a979 Mon Sep 17 00:00:00 2001 From: terrill <314156936@qq.com> Date: Fri, 27 Jul 2018 10:11:38 +0800 Subject: [PATCH 12/18] one_time_key inner multiple bug --- .../dendrite/encryptoapi/storage/encrypt_keys_table.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go b/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go index 8c822fbd2..94bbce6a8 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go @@ -155,10 +155,10 @@ func (s *keyStatements) selectSingleKey( &holder.KeyAlgorithm, &holder.Signature, ); err != nil { - deleteStmt := s.deleteSingleKeyStmt - _, err = deleteStmt.ExecContext(ctx, userID, deviceID, algorithm, holder.KeyID) return holder, err } + deleteStmt := s.deleteSingleKeyStmt + _, err = deleteStmt.ExecContext(ctx, userID, deviceID, algorithm, holder.KeyID) return holder, err } From 9d391f54f2484d928d781f8917e458528cc373e6 Mon Sep 17 00:00:00 2001 From: terrill <314156936@qq.com> Date: Fri, 27 Jul 2018 10:21:54 +0800 Subject: [PATCH 13/18] revision last conflict --- .../matrix-org/dendrite/encryptoapi/routing/routing.go | 6 +++--- .../matrix-org/dendrite/syncapi/routing/routing.go | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go b/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go index 4ec1b184d..20c3b273f 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go @@ -37,13 +37,13 @@ func Setup( unstablemux := apiMux.PathPrefix(pathPrefixUnstable).Subrouter() unstablemux.Handle("/keys/upload/{deviceID}", - common.MakeAuthAPI("upload keys", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + common.MakeAuthAPI("upload keys", nil, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return UploadPKeys(req, encryptionDB, device.UserID, device.ID) }), ).Methods(http.MethodPost, http.MethodOptions) unstablemux.Handle("/keys/upload", - common.MakeAuthAPI("upload keys", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + common.MakeAuthAPI("upload keys", nil, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return UploadPKeys(req, encryptionDB, device.UserID, device.ID) }), ).Methods(http.MethodPost, http.MethodOptions) @@ -56,7 +56,7 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) unstablemux.Handle("/keys/claim", - common.MakeAuthAPI("claim keys", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + common.MakeAuthAPI("claim keys", nil, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return ClaimOneTimeKeys(req, encryptionDB) }), ).Methods(http.MethodPost, http.MethodOptions) diff --git a/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go b/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go index 10d650a20..f4e6fa959 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go @@ -34,21 +34,21 @@ func Setup(apiMux *mux.Router, srp *sync.RequestPool, syncDB *storage.SyncServer r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() unstablemux := apiMux.PathPrefix(pathPrefixUnstable).Subrouter() - r0mux.Handle("/sync", common.MakeAuthAPI("sync", nil, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + r0mux.Handle("/sync", common.MakeAuthAPI("sync", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return srp.OnIncomingSyncRequest(req, device) })).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/state", common.MakeAuthAPI("room_state", nil, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + r0mux.Handle("/rooms/{roomID}/state", common.MakeAuthAPI("room_state", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { vars := mux.Vars(req) return OnIncomingStateRequest(req, syncDB, vars["roomID"]) })).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/state/{type}", common.MakeAuthAPI("room_state", nil, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + r0mux.Handle("/rooms/{roomID}/state/{type}", common.MakeAuthAPI("room_state", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { vars := mux.Vars(req) return OnIncomingStateTypeRequest(req, syncDB, vars["roomID"], vars["type"], "") })).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", common.MakeAuthAPI("room_state", nil, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", common.MakeAuthAPI("room_state", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { vars := mux.Vars(req) return OnIncomingStateTypeRequest(req, syncDB, vars["roomID"], vars["type"], vars["stateKey"]) })).Methods(http.MethodGet, http.MethodOptions) From ce31a223d8cd74bff5243256aec16b6e59e55a62 Mon Sep 17 00:00:00 2001 From: terrill <314156936@qq.com> Date: Fri, 27 Jul 2018 10:32:51 +0800 Subject: [PATCH 14/18] linton --- .../matrix-org/dendrite/encryptoapi/routing/routing.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go b/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go index 20c3b273f..4ec1b184d 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go @@ -37,13 +37,13 @@ func Setup( unstablemux := apiMux.PathPrefix(pathPrefixUnstable).Subrouter() unstablemux.Handle("/keys/upload/{deviceID}", - common.MakeAuthAPI("upload keys", nil, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + common.MakeAuthAPI("upload keys", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return UploadPKeys(req, encryptionDB, device.UserID, device.ID) }), ).Methods(http.MethodPost, http.MethodOptions) unstablemux.Handle("/keys/upload", - common.MakeAuthAPI("upload keys", nil, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + common.MakeAuthAPI("upload keys", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return UploadPKeys(req, encryptionDB, device.UserID, device.ID) }), ).Methods(http.MethodPost, http.MethodOptions) @@ -56,7 +56,7 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) unstablemux.Handle("/keys/claim", - common.MakeAuthAPI("claim keys", nil, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + common.MakeAuthAPI("claim keys", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return ClaimOneTimeKeys(req, encryptionDB) }), ).Methods(http.MethodPost, http.MethodOptions) From 0c682d16aeed62b8e49e46e84bd26fc2384425ca Mon Sep 17 00:00:00 2001 From: terrill <314156936@qq.com> Date: Fri, 27 Jul 2018 11:04:09 +0800 Subject: [PATCH 15/18] linton --- .../matrix-org/dendrite/encryptoapi/routing/routing.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go b/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go index 4ec1b184d..9131345f0 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go @@ -24,6 +24,7 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/encryptoapi/storage" "github.com/matrix-org/util" + "github.com/matrix-org/dendrite/clientapi/auth" ) const pathPrefixUnstable = "/_matrix/client/unstable" @@ -34,29 +35,30 @@ func Setup( encryptionDB *storage.Database, deviceDB *devices.Database, ) { + dataDB := auth.DeviceDatabase(deviceDB) unstablemux := apiMux.PathPrefix(pathPrefixUnstable).Subrouter() unstablemux.Handle("/keys/upload/{deviceID}", - common.MakeAuthAPI("upload keys", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + common.MakeAuthAPI("upload keys", dataDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return UploadPKeys(req, encryptionDB, device.UserID, device.ID) }), ).Methods(http.MethodPost, http.MethodOptions) unstablemux.Handle("/keys/upload", - common.MakeAuthAPI("upload keys", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + common.MakeAuthAPI("upload keys", dataDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return UploadPKeys(req, encryptionDB, device.UserID, device.ID) }), ).Methods(http.MethodPost, http.MethodOptions) unstablemux.Handle("/keys/query", - common.MakeAuthAPI("query keys", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + common.MakeAuthAPI("query keys", dataDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { //vars := mux.Vars(req) return QueryPKeys(req, encryptionDB, device.ID, deviceDB) }), ).Methods(http.MethodPost, http.MethodOptions) unstablemux.Handle("/keys/claim", - common.MakeAuthAPI("claim keys", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + common.MakeAuthAPI("claim keys", dataDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return ClaimOneTimeKeys(req, encryptionDB) }), ).Methods(http.MethodPost, http.MethodOptions) From 9c36971b117dc0dc25b90cc44d6e66fb37b3e309 Mon Sep 17 00:00:00 2001 From: terrill <314156936@qq.com> Date: Fri, 27 Jul 2018 11:20:02 +0800 Subject: [PATCH 16/18] a slight change to match master auth.Data --- .../matrix-org/dendrite/encryptoapi/routing/routing.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go b/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go index 9131345f0..1cb78238d 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go @@ -35,30 +35,30 @@ func Setup( encryptionDB *storage.Database, deviceDB *devices.Database, ) { - dataDB := auth.DeviceDatabase(deviceDB) + authData := auth.Data{nil, deviceDB, nil} unstablemux := apiMux.PathPrefix(pathPrefixUnstable).Subrouter() unstablemux.Handle("/keys/upload/{deviceID}", - common.MakeAuthAPI("upload keys", dataDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + common.MakeAuthAPI("upload keys", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return UploadPKeys(req, encryptionDB, device.UserID, device.ID) }), ).Methods(http.MethodPost, http.MethodOptions) unstablemux.Handle("/keys/upload", - common.MakeAuthAPI("upload keys", dataDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + common.MakeAuthAPI("upload keys", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return UploadPKeys(req, encryptionDB, device.UserID, device.ID) }), ).Methods(http.MethodPost, http.MethodOptions) unstablemux.Handle("/keys/query", - common.MakeAuthAPI("query keys", dataDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + common.MakeAuthAPI("query keys", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { //vars := mux.Vars(req) return QueryPKeys(req, encryptionDB, device.ID, deviceDB) }), ).Methods(http.MethodPost, http.MethodOptions) unstablemux.Handle("/keys/claim", - common.MakeAuthAPI("claim keys", dataDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + common.MakeAuthAPI("claim keys", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return ClaimOneTimeKeys(req, encryptionDB) }), ).Methods(http.MethodPost, http.MethodOptions) From fcf90dc5247fb1bfadf3b721b283a057fb20f821 Mon Sep 17 00:00:00 2001 From: terrill <314156936@qq.com> Date: Fri, 27 Jul 2018 11:27:24 +0800 Subject: [PATCH 17/18] a slight change to match master auth.Data --- .../matrix-org/dendrite/encryptoapi/routing/routing.go | 2 +- src/github.com/matrix-org/dendrite/syncapi/routing/routing.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go b/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go index 1cb78238d..eeb6d4ba0 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go @@ -21,10 +21,10 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" + "github.com/matrix-org/dendrite/clientapi/auth" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/encryptoapi/storage" "github.com/matrix-org/util" - "github.com/matrix-org/dendrite/clientapi/auth" ) const pathPrefixUnstable = "/_matrix/client/unstable" diff --git a/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go b/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go index ebd3e6cab..dc764f31f 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go @@ -58,7 +58,7 @@ func Setup(apiMux *mux.Router, srp *sync.RequestPool, syncDB *storage.SyncServer })).Methods(http.MethodGet, http.MethodOptions) unstablemux.Handle("/sendToDevice/{eventType}/{txnId}", - common.MakeAuthAPI("look up changes", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + common.MakeAuthAPI("look up changes", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { vars := mux.Vars(req) eventType := vars["eventType"] txnID := vars["txnId"] From 8b4b3c6fc46900e9bfe5e234eda309200662b34a Mon Sep 17 00:00:00 2001 From: terrill <314156936@qq.com> Date: Fri, 27 Jul 2018 13:02:54 +0800 Subject: [PATCH 18/18] key_count extension for /sync --- .../cmd/dendrite-monolith-server/main.go | 7 +- .../cmd/dendrite-sync-api-server/main.go | 2 +- .../dendrite/encryptoapi/encryptoapi.go | 3 +- .../encryptoapi/storage/encrypt_keys_table.go | 45 +++++++++++-- .../dendrite/encryptoapi/storage/storage.go | 9 +++ .../dendrite/syncapi/routing/routing.go | 5 +- .../dendrite/syncapi/sync/keyextension.go | 64 +++++++++++++++++++ .../dendrite/syncapi/sync/requestpool.go | 4 +- .../matrix-org/dendrite/syncapi/syncapi.go | 4 +- .../dendrite/syncapi/types/types.go | 3 +- 10 files changed, 129 insertions(+), 17 deletions(-) create mode 100644 src/github.com/matrix-org/dendrite/syncapi/sync/keyextension.go diff --git a/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go b/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go index 9ea37b665..06338d374 100644 --- a/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go @@ -22,7 +22,6 @@ import ( "github.com/matrix-org/dendrite/common/transactions" "github.com/matrix-org/dendrite/typingserver" - "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/clientapi" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/basecomponent" @@ -59,7 +58,7 @@ func main() { alias, input, query := roomserver.SetupRoomServerComponent(base) typingInputAPI := typingserver.SetupTypingServerComponent(base) - encryptoapi.SetupEcryptoapi(base, deviceDB) + encryptDB := encryptoapi.SetupEcryptoapi(base, deviceDB) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, @@ -70,8 +69,8 @@ func main() { federationsender.SetupFederationSenderComponent(base, federation, query) mediaapi.SetupMediaAPIComponent(base, deviceDB) publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB) - syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query) - appservice.SetupAppServiceAPIComponent(base, accountDB, deviceDB, federation, alias, query, transactions.New()) + syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, encryptDB) + //appservice.SetupAppServiceAPIComponent(base, accountDB, deviceDB, federation, alias, query, transactions.New()) httpHandler := common.WrapHandlerInCORS(base.APIMux) diff --git a/src/github.com/matrix-org/dendrite/cmd/dendrite-sync-api-server/main.go b/src/github.com/matrix-org/dendrite/cmd/dendrite-sync-api-server/main.go index 343d3567d..82b754447 100644 --- a/src/github.com/matrix-org/dendrite/cmd/dendrite-sync-api-server/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/dendrite-sync-api-server/main.go @@ -29,7 +29,7 @@ func main() { _, _, query := base.CreateHTTPRoomserverAPIs() - syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query) + syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, nil) base.SetupAndServeHTTP(string(base.Cfg.Listen.SyncAPI)) } diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go b/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go index d4529a27c..dca5d42e0 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go @@ -31,7 +31,7 @@ import ( func SetupEcryptoapi( base *basecomponent.BaseDendrite, deviceDB *devices.Database, -) { +) *storage.Database { encryptionDB, err := storage.NewDatabase(string(base.Cfg.Database.EncryptAPI)) if err != nil { logrus.WithError(err).Panicf("failed to connect to encryption db") @@ -42,4 +42,5 @@ func SetupEcryptoapi( deviceDB, ) routing.InitNotifier(base) + return encryptionDB } diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go b/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go index 94bbce6a8..97cb6196a 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go @@ -59,14 +59,19 @@ const selectAllkeysSQL = ` SELECT user_id, device_id, key_id, key_type, key_info, algorithm, signature FROM encrypt_keys WHERE user_id = $1 AND key_type = $2 ` +const selectCountOneTimeKey = ` +SELECT algorithm, COUNT(algorithm) FROM encrypt_keys WHERE user_id = $1 AND device_id = $2 AND key_type = 'one_time_key' +GROUP BY algorithm +` type keyStatements struct { - insertKeyStmt *sql.Stmt - selectKeyStmt *sql.Stmt - selectInKeysStmt *sql.Stmt - selectAllKeyStmt *sql.Stmt - selectSingleKeyStmt *sql.Stmt - deleteSingleKeyStmt *sql.Stmt + insertKeyStmt *sql.Stmt + selectKeyStmt *sql.Stmt + selectInKeysStmt *sql.Stmt + selectAllKeyStmt *sql.Stmt + selectSingleKeyStmt *sql.Stmt + deleteSingleKeyStmt *sql.Stmt + selectCountOneTimeKeyStmt *sql.Stmt } func (s *keyStatements) prepare(db *sql.DB) (err error) { @@ -92,6 +97,9 @@ func (s *keyStatements) prepare(db *sql.DB) (err error) { if s.selectSingleKeyStmt, err = db.Prepare(deleteSinglekeySQL); err != nil { return } + if s.selectCountOneTimeKeyStmt, err = db.Prepare(selectCountOneTimeKey); err != nil { + return + } return } @@ -216,3 +224,28 @@ func injectKeyHolder(rows *sql.Rows, keyHolder []types.KeyHolder) (holders []typ holders = keyHolder return } + +// select by user and device +func (s *keyStatements) selectOneTimeKeyCount( + ctx context.Context, + userID, deviceID string, +) (map[string]int, error) { + holders := make(map[string]int) + rows, err := s.selectCountOneTimeKeyStmt.QueryContext(ctx, userID, deviceID) + if err != nil { + return nil, err + } + for rows.Next() { + var singleKey string + var singleVal int + if err = rows.Scan( + &singleKey, + &singleVal, + ); err != nil { + return nil, err + } + holders[singleKey] = singleVal + } + err = rows.Close() + return holders, err +} diff --git a/src/github.com/matrix-org/dendrite/encryptoapi/storage/storage.go b/src/github.com/matrix-org/dendrite/encryptoapi/storage/storage.go index 2ed567a7d..372221630 100644 --- a/src/github.com/matrix-org/dendrite/encryptoapi/storage/storage.go +++ b/src/github.com/matrix-org/dendrite/encryptoapi/storage/storage.go @@ -120,3 +120,12 @@ func (d *Database) SelectOneTimeKeySingle( holder, err = d.keyStatements.selectSingleKey(ctx, userID, deviceID, algorithm) return } + +// SyncOneTimeCount for sync device_one_time_keys_count extension +func (d *Database) SyncOneTimeCount( + ctx context.Context, + userID, deviceID string, +) (holder map[string]int, err error) { + holder, err = d.keyStatements.selectOneTimeKeyCount(ctx, userID, deviceID) + return +} diff --git a/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go b/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go index dc764f31f..aa5d7034d 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/common" + encryptoapi "github.com/matrix-org/dendrite/encryptoapi/storage" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/sync" "github.com/matrix-org/util" @@ -31,7 +32,7 @@ const pathPrefixR0 = "/_matrix/client/r0" const pathPrefixUnstable = "/_matrix/client/unstable" // Setup configures the given mux with sync-server listeners -func Setup(apiMux *mux.Router, srp *sync.RequestPool, syncDB *storage.SyncServerDatabase, deviceDB *devices.Database, notifier *sync.Notifier) { +func Setup(apiMux *mux.Router, srp *sync.RequestPool, syncDB *storage.SyncServerDatabase, deviceDB *devices.Database, notifier *sync.Notifier, encryptDB *encryptoapi.Database) { r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() unstablemux := apiMux.PathPrefix(pathPrefixUnstable).Subrouter() @@ -39,7 +40,7 @@ func Setup(apiMux *mux.Router, srp *sync.RequestPool, syncDB *storage.SyncServer // TODO: Add AS support for all handlers below. r0mux.Handle("/sync", common.MakeAuthAPI("sync", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - return srp.OnIncomingSyncRequest(req, device) + return srp.OnIncomingSyncRequest(req, device, encryptDB) })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { diff --git a/src/github.com/matrix-org/dendrite/syncapi/sync/keyextension.go b/src/github.com/matrix-org/dendrite/syncapi/sync/keyextension.go new file mode 100644 index 000000000..7643ec797 --- /dev/null +++ b/src/github.com/matrix-org/dendrite/syncapi/sync/keyextension.go @@ -0,0 +1,64 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "context" + encryptoapi "github.com/matrix-org/dendrite/encryptoapi/storage" + "github.com/matrix-org/dendrite/syncapi/types" + "sync" +) + +type keyCounter struct { + sync.RWMutex + m map[string]map[string]int +} + +var counter = keyCounter{ + m: make(map[string]map[string]int), +} + +// CounterRead returns uid to countMap +func CounterRead(uid string) map[string]int { + counter.RLock() + defer counter.RUnlock() + return counter.m[uid] +} + +// CounterWrite write count map to share for all response +func CounterWrite(uid string, m map[string]int) { + counter.Lock() + defer counter.Unlock() + counter.m[uid] = m +} + +// KeyCountEXT key count extension process +func KeyCountEXT( + ctx context.Context, + encryptionDB *encryptoapi.Database, + respIn types.Response, + userID, deviceID string, +) (respOut *types.Response) { + + respOut = &respIn + // when extension works at the very beginning + resp, err := encryptionDB.SyncOneTimeCount(ctx, userID, deviceID) + CounterWrite(userID, resp) + if err != nil { + return + } + respOut.SignNum = resp + return +} diff --git a/src/github.com/matrix-org/dendrite/syncapi/sync/requestpool.go b/src/github.com/matrix-org/dendrite/syncapi/sync/requestpool.go index 7092a09b3..51e60e348 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/sync/requestpool.go +++ b/src/github.com/matrix-org/dendrite/syncapi/sync/requestpool.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" + encryptoapi "github.com/matrix-org/dendrite/encryptoapi/storage" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -44,7 +45,7 @@ func NewRequestPool(db *storage.SyncServerDatabase, n *Notifier, adb *accounts.D // OnIncomingSyncRequest is called when a client makes a /sync request. This function MUST be // called in a dedicated goroutine for this request. This function will block the goroutine // until a response is ready, or it times out. -func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtypes.Device) util.JSONResponse { +func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtypes.Device, encryptDB *encryptoapi.Database) util.JSONResponse { // Extract values from request logger := util.GetLogger(req.Context()) userID := device.UserID @@ -109,6 +110,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype // std extension consideration syncData = storage.StdEXT(syncReq.ctx, rp.db, *syncData, syncReq.device.UserID, syncReq.device.ID, int64(currPos)) + syncData = KeyCountEXT(syncReq.ctx, encryptDB, *syncData, syncReq.device.UserID, syncReq.device.ID) if err != nil { return httputil.LogThenError(req, err) diff --git a/src/github.com/matrix-org/dendrite/syncapi/syncapi.go b/src/github.com/matrix-org/dendrite/syncapi/syncapi.go index c1de63c2b..c1790f8d1 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/syncapi.go +++ b/src/github.com/matrix-org/dendrite/syncapi/syncapi.go @@ -24,6 +24,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" + encryptoapi "github.com/matrix-org/dendrite/encryptoapi/storage" "github.com/matrix-org/dendrite/syncapi/consumers" "github.com/matrix-org/dendrite/syncapi/routing" "github.com/matrix-org/dendrite/syncapi/storage" @@ -38,6 +39,7 @@ func SetupSyncAPIComponent( deviceDB *devices.Database, accountsDB *accounts.Database, queryAPI api.RoomserverQueryAPI, + encryptDB *encryptoapi.Database, ) { syncDB, err := storage.NewSyncServerDatabase(string(base.Cfg.Database.SyncAPI)) if err != nil { @@ -71,5 +73,5 @@ func SetupSyncAPIComponent( logrus.WithError(err).Panicf("failed to start client data consumer") } - routing.Setup(base.APIMux, requestPool, syncDB, deviceDB, notifier) + routing.Setup(base.APIMux, requestPool, syncDB, deviceDB, notifier, encryptDB) } diff --git a/src/github.com/matrix-org/dendrite/syncapi/types/types.go b/src/github.com/matrix-org/dendrite/syncapi/types/types.go index e8f4e02a5..a86d7a38d 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/types/types.go +++ b/src/github.com/matrix-org/dendrite/syncapi/types/types.go @@ -50,7 +50,8 @@ type Response struct { Invite map[string]InviteResponse `json:"invite"` Leave map[string]LeaveResponse `json:"leave"` } `json:"rooms"` - ToDevice ToDevice `json:"to_device"` + ToDevice ToDevice `json:"to_device"` + SignNum map[string]int `json:"device_one_time_keys_count"` } // StdHolder represents send to device response from db