standardize format

dendrite-config.yaml

@@ -87,6 +87,7 @@ kafka:
         output_room_event: roomserverOutput
         output_client_data: clientapiOutput
         user_updates: userUpdates
+        keyUpdate: keyUpdate
 
 # The postgres connection configs for connecting to the databases e.g a postgres:// URI
 database:
@@ -98,6 +99,7 @@ database:
     server_key: "postgres://dendrite:itsasecret@localhost/dendrite_serverkey?sslmode=disable"
     federation_sender: "postgres://dendrite:itsasecret@localhost/dendrite_federationsender?sslmode=disable"
     public_rooms_api: "postgres://dendrite:itsasecret@localhost/dendrite_publicroomsapi?sslmode=disable"
+    encrypt_api: "postgres:////dendrite:itsasecret@localhost/dendrite_encryptapi?sslmode=disable"
     # If using naffka you need to specify a naffka database
     # naffka: "postgres://dendrite:itsasecret@localhost/dendrite_naffka?sslmode=disable"
 

docker/postgres/create_db.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 
-for db in account device mediaapi syncapi roomserver serverkey federationsender publicroomsapi naffka; do
+for db in account device mediaapi syncapi roomserver serverkey federationsender publicroomsapi naffka encryptapi; do
     createdb -O dendrite dendrite_$db
 done

src/github.com/matrix-org/dendrite/clientapi/auth/authtypes/device.go

@@ -22,4 +22,5 @@ type Device struct {
 	// This uniquely identifies the device from all other devices and clients.
 	AccessToken string
 	// TODO: display name, last used timestamp, keys, etc
+	DisplayName string
 }

src/github.com/matrix-org/dendrite/common/config/config.go

@@ -167,6 +167,8 @@ type Dendrite struct {
 		PublicRoomsAPI DataSource `yaml:"public_rooms_api"`
 		// The Naffka database is used internally by the naffka library, if used.
 		Naffka DataSource `yaml:"naffka,omitempty"`
+		// Encryption api database
+		EncryptAPI DataSource `yaml:"encrypt_api"`
 	} `yaml:"database"`
 
 	// TURN Server Config

src/github.com/matrix-org/dendrite/encryptoapi/encryptoapi.go

@@ -0,0 +1,44 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package encryptoapi
+
+import (
+	"github.com/matrix-org/dendrite/common/basecomponent"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/encryptoapi/routing"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
+	"github.com/matrix-org/dendrite/encryptoapi/storage"
+	"fmt"
+)
+
+func SetupEcryptoapi(
+	base *basecomponent.BaseDendrite,
+	accountsDB *accounts.Database,
+	deviceDB *devices.Database,
+
+) {
+
+	encryptionDB, err := storage.NewDatabase(string(base.Cfg.Database.EncryptAPI))
+	fmt.Print(err)
+	routing.Setup(
+		base.APIMux,
+		*base.Cfg,
+		encryptionDB,
+		accountsDB,
+		deviceDB,
+	)
+	routing.InitNotifier(base)
+
+}

src/github.com/matrix-org/dendrite/encryptoapi/routing/keys.go

@@ -0,0 +1,405 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"github.com/matrix-org/util"
+	"github.com/matrix-org/dendrite/encryptoapi/storage"
+	"net/http"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/encryptoapi/types"
+	"context"
+	"github.com/pkg/errors"
+	"strings"
+	"fmt"
+	"encoding/json"
+	"time"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/dendrite/common/basecomponent"
+	"github.com/Shopify/sarama"
+)
+
+const (
+	TYPESUM          = iota
+	TYPECLAIM
+	TYPEVAL
+	BODYDEVICEKEY
+	BODYONETIMEKEY
+	ONETIMEKEYSTRING
+	ONETIMEKEYOBJECT
+)
+
+type KeyNotifier struct {
+	base *basecomponent.BaseDendrite
+	ch   sarama.AsyncProducer
+}
+
+var keyProducer = &KeyNotifier{}
+
+func UploadPKeys(req *http.Request, encryptionDB *storage.Database, userID, deviceID string) util.JSONResponse {
+	var keybody types.UploadEncrypt
+	if reqErr := httputil.UnmarshalJSONRequest(req, &keybody); reqErr != nil {
+		return *reqErr
+	}
+	keySpecific := turnSpecific(keybody)
+	err := persistKeys(encryptionDB, req.Context(), &keySpecific, userID, deviceID)
+	numMap := (QueryOneTimeKeys(
+		TYPESUM,
+		userID,
+		deviceID,
+		encryptionDB,
+		req.Context())).(map[string]int)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadGateway,
+			JSON: types.UploadResponse{
+				Count: numMap,
+			},
+		}
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: types.UploadResponse{
+			Count: numMap,
+		},
+	}
+}
+
+func QueryPKeys(req *http.Request, encryptionDB *storage.Database, userID, deviceID string, deviceDB *devices.Database) util.JSONResponse {
+	var queryRq types.QueryRequest
+	queryRp := types.QueryResponse{}
+	queryRp.Failure = make(map[string]interface{})
+	queryRp.DeviceKeys = make(map[string]map[string]types.DeviceKeysQuery)
+	if reqErr := httputil.UnmarshalJSONRequest(req, &queryRq); reqErr != nil {
+		return *reqErr
+	}
+
+	/*
+		federation consideration: when user id is in federation, a query is needed to ask fed for keys
+		domain --------+ fed (keys)
+		domain +--tout-- timer
+	*/
+	// todo: Add federation processing at specific userID.
+	if false /*federation judgement*/ {
+		tout := queryRq.Timeout
+		if tout == 0 {
+			tout = int64(10 * time.Second)
+		}
+		stimuCh := make(chan int)
+		go func() {
+			time.Sleep(time.Duration(tout) * 1000 * 1000)
+			close(stimuCh)
+		}()
+		select {
+		case <-stimuCh:
+			queryRp.Failure = make(map[string]interface{})
+			// todo: key in this map is restricted to username at the end, yet a mocked one.
+			queryRp.Failure["@alice:localhost"] = "ran out of offered time"
+		}
+	}
+
+	for uid, arr := range queryRq.DeviceKeys {
+		queryRp.DeviceKeys[uid] = make(map[string]types.DeviceKeysQuery)
+		deviceKeysQueryMap := queryRp.DeviceKeys[uid]
+		// backward compatible to old interface
+		midArr := []string{}
+		for device, _ := range arr.(map[string]interface{}) {
+			midArr = append(midArr, device)
+		}
+		dkeys, _ := encryptionDB.QueryInRange(req.Context(), uid, midArr)
+		for _, key := range dkeys {
+			// preset for complicated nested map struct
+			if _, ok := deviceKeysQueryMap[key.Device_id]; !ok {
+				// make consistency
+				deviceKeysQueryMap[key.Device_id] = types.DeviceKeysQuery{}
+			}
+			if deviceKeysQueryMap[key.Device_id].Signature == nil {
+				mid := make(map[string]map[string]string)
+				midmap := deviceKeysQueryMap[key.Device_id]
+				midmap.Signature = mid
+				deviceKeysQueryMap[key.Device_id] = midmap
+			}
+			if deviceKeysQueryMap[key.Device_id].Keys == nil {
+				mid := make(map[string]string)
+				midmap := deviceKeysQueryMap[key.Device_id]
+				midmap.Keys = mid
+				deviceKeysQueryMap[key.Device_id] = midmap
+			}
+			if _, ok := deviceKeysQueryMap[key.Device_id].Signature[uid]; !ok {
+				// make consistency
+				deviceKeysQueryMap[key.Device_id].Signature[uid] = make(map[string]string)
+			}
+			// load for accomplishment
+			single := deviceKeysQueryMap[key.Device_id]
+
+			resKey := fmt.Sprintf("@%s:%s", key.Key_algorithm, key.Device_id)
+			resBody := key.Key
+			if _, ok := single.Keys[resKey]; !ok {
+			}
+			single.Keys[resKey] = resBody
+			single.DeviceId = key.Device_id
+			single.UserId = key.User_id
+			single.Signature[uid][fmt.Sprintf("@%s:%s", "ed25519", key.Device_id)] = key.Signature
+			single.Algorithm, _ = takeAL(*encryptionDB, req.Context(), key.User_id, key.Device_id)
+			localpart, _, _ := gomatrixserverlib.SplitID('@', uid)
+			device, _ := deviceDB.GetDeviceByID(req.Context(), localpart, deviceID)
+			single.Unsigned.Info = device.DisplayName
+			deviceKeysQueryMap[key.Device_id] = single
+		}
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: queryRp,
+	}
+}
+
+func ClaimOneTimeKeys(req *http.Request, encryptionDB *storage.Database, userID, deviceID string, deviceDB *devices.Database) util.JSONResponse {
+	var claimRq types.ClaimRequest
+	claimRp := types.ClaimResponse{}
+	claimRp.Failures = make(map[string]interface{})
+	claimRp.ClaimBody = make(map[string]map[string]map[string]interface{})
+	if reqErr := httputil.UnmarshalJSONRequest(req, &claimRq); reqErr != nil {
+		return *reqErr
+	}
+
+	/*
+		federation consideration: when user id is in federation, a query is needed to ask fed for keys
+		domain --------+ fed (keys)
+		domain +--tout-- timer
+	*/
+	// todo: Add federation processing at specific userID.
+	if false /*federation judgement*/ {
+		tout := claimRq.Timeout
+		stimuCh := make(chan int)
+		go func() {
+			time.Sleep(time.Duration(tout) * 1000 * 1000)
+			close(stimuCh)
+		}()
+		select {
+		case <-stimuCh:
+			claimRp.Failures = make(map[string]interface{})
+			// todo: key in this map is restricted to username at the end, yet a mocked one.
+			claimRp.Failures["@alice:localhost"] = "ran out of offered time"
+		}
+	}
+
+	content := claimRq.ClaimDetail
+	for uid, detail := range content {
+		for deviceID, al := range detail {
+			var alTyp int
+			if strings.Contains(al, "signed") {
+				alTyp = ONETIMEKEYOBJECT
+			} else {
+				alTyp = ONETIMEKEYSTRING
+			}
+			key, err := pickOne(*encryptionDB, req.Context(), uid, deviceID, al)
+			if err != nil {
+				claimRp.Failures[uid] = fmt.Sprintf("%s:%s", "fail to get keys for device ", deviceID)
+			}
+			claimRp.ClaimBody[uid] = make(map[string]map[string]interface{})
+			keymap := claimRp.ClaimBody[uid][deviceID]
+			keymap = make(map[string]interface{})
+			switch alTyp {
+			case ONETIMEKEYSTRING:
+				keymap[fmt.Sprintf("%s:%s", al, key.Key_id)] = key.Key
+			case ONETIMEKEYOBJECT:
+				sig := make(map[string]map[string]string)
+				sig[uid] = make(map[string]string)
+				sig[uid][fmt.Sprintf("%s:%s", "ed25519", deviceID)] = key.Signature
+				keymap[fmt.Sprintf("%s:%s", al, key.Key_id)] = types.KeyObject{Key: key.Key, Signature: sig}
+			}
+			claimRp.ClaimBody[uid][deviceID] = keymap
+		}
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: claimRp,
+	}
+}
+
+func LookUpChangedPKeys() util.JSONResponse {
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}
+
+// todo: check through interface for duplicate
+func checkUpload(req *types.UploadEncryptSpecific, typ int) bool {
+	if typ == BODYDEVICEKEY {
+		devicekey := req.DeviceKeys
+		if devicekey.UserId == "" {
+			return false
+		}
+	}
+	if typ == BODYONETIMEKEY {
+		if req.OneTimeKey.KeyString == nil || req.OneTimeKey.KeyObject == nil {
+			return false
+		}
+	}
+	return true
+}
+
+// todo: complete this field through claim type
+func QueryOneTimeKeys(
+	typ int,
+	userID, deviceID string,
+	encryptionDB *storage.Database,
+	ctx context.Context,
+) interface{} {
+	if typ == TYPESUM {
+		res, _ := encryptionDB.SelectOneTimeKeyCount(ctx, deviceID, userID)
+		return res
+	}
+	return nil
+}
+
+// todo: complete this function and invoke through sign out extension or some scenarios else those matter
+// when web client sign out, a clean should be processed, cause all keys would never been used from then on.
+func ClearUnused() {}
+
+func persistKeys(
+	database *storage.Database,
+	ctx context.Context,
+	body *types.UploadEncryptSpecific,
+	userID,
+	deviceID string,
+) (err error) {
+	// in order to persist keys , a check filtering duplicate should be processed
+	if checkUpload(body, BODYDEVICEKEY) {
+		deviceKeys := body.DeviceKeys
+		al := deviceKeys.Algorithm
+		err = persistAl(*database, ctx, userID, deviceID, al)
+		if checkUpload(body, BODYONETIMEKEY) {
+			// insert one time keys firstly
+			onetimeKeys := body.OneTimeKey
+			for al_keyID, val := range onetimeKeys.KeyString {
+				al := (strings.Split(al_keyID, ":"))[0]
+				keyID := (strings.Split(al_keyID, ":"))[1]
+				keyInfo := val
+				keyTyp := "one_time_key"
+				sig := ""
+				database.InsertKey(ctx, deviceID, userID, keyID, keyTyp, keyInfo, al, sig)
+			}
+			for al_keyID, val := range onetimeKeys.KeyObject {
+				al := (strings.Split(al_keyID, ":"))[0]
+				keyID := (strings.Split(al_keyID, ":"))[1]
+				keyInfo := val.Key
+				keyTyp := "one_time_key"
+				sig := val.Signature[userID][fmt.Sprintf("%s:%s", "ed25519", deviceID)]
+				database.InsertKey(ctx, deviceID, userID, keyID, keyTyp, keyInfo, al, sig)
+			}
+			// insert device keys
+			keys := deviceKeys.Keys
+			sigs := deviceKeys.Signature
+			for al_device, key := range keys {
+				al := (strings.Split(al_device, ":"))[0]
+				keyTyp := "device_key"
+				keyInfo := key
+				keyID := ""
+				sig := sigs[userID][fmt.Sprintf("%s:%s", "ed25519", deviceID)]
+				database.InsertKey(
+					ctx, deviceID, userID, keyID, keyTyp, keyInfo, al, sig)
+			}
+		} else {
+			keys := deviceKeys.Keys
+			sigs := deviceKeys.Signature
+			for al_device, key := range keys {
+				al := (strings.Split(al_device, ":"))[0]
+				keyTyp := "device_key"
+				keyInfo := key
+				keyID := ""
+				sig := sigs[userID][fmt.Sprintf("%s:%s", "ed25519", deviceID)]
+				database.InsertKey(ctx, deviceID, userID, keyID, keyTyp, keyInfo, al, sig)
+			}
+		}
+		// notifier to sync server
+		upnotify(userID)
+	} else {
+		if checkUpload(body, BODYONETIMEKEY) {
+			onetimeKeys := body.OneTimeKey
+			for al_keyID, val := range onetimeKeys.KeyString {
+				al := (strings.Split(al_keyID, ":"))[0]
+				keyID := (strings.Split(al_keyID, ":"))[1]
+				keyInfo := val
+				keyTyp := "one_time_key"
+				sig := ""
+				database.InsertKey(ctx, deviceID, userID, keyID, keyTyp, keyInfo, al, sig)
+			}
+			for al_keyID, val := range onetimeKeys.KeyObject {
+				al := (strings.Split(al_keyID, ":"))[0]
+				keyID := (strings.Split(al_keyID, ":"))[1]
+				keyInfo := val.Key
+				keyTyp := "one_time_key"
+				sig := val.Signature[userID][fmt.Sprintf("%s:%s", "ed25519", deviceID)]
+				database.InsertKey(ctx, deviceID, userID, keyID, keyTyp, keyInfo, al, sig)
+			}
+		} else {
+			return errors.New("Fail to touch keys !")
+		}
+	}
+	return err
+}
+
+func turnSpecific(cont types.UploadEncrypt) (spec types.UploadEncryptSpecific) {
+	// both device keys are coordinate
+	spec.DeviceKeys = cont.DeviceKeys
+	spec.OneTimeKey.KeyString = make(map[string]string)
+	spec.OneTimeKey.KeyObject = make(map[string]types.KeyObject)
+	mapStringInterface := cont.OneTimeKey
+	for key, val := range mapStringInterface {
+		value, ok := val.(string)
+		if ok {
+			spec.OneTimeKey.KeyString[key] = value
+		} else {
+			valueObject := types.KeyObject{}
+			target, _ := json.Marshal(val)
+			json.Unmarshal(target, &valueObject)
+			spec.OneTimeKey.KeyObject[key] = valueObject
+		}
+	}
+	return
+}
+
+func persistAl(encryptDB storage.Database, ctx context.Context, uid, device string, al []string) (err error) {
+	err = encryptDB.InsertAl(ctx, uid, device, al)
+	return
+}
+
+func takeAL(encryptDB storage.Database, ctx context.Context, uid, device string) (al []string, err error) {
+	al, err = encryptDB.SelectAl(ctx, uid, device)
+	return
+}
+
+func pickOne(encryptDB storage.Database, ctx context.Context, uid, device, al string) (key types.KeyHolder, err error) {
+	key, err = encryptDB.SelectOneTimeKeySingle(ctx, uid, device, al)
+	return
+}
+
+func upnotify(userID string) {
+	m := sarama.ProducerMessage{
+		Topic: "keyUpdate",
+		Key:   sarama.StringEncoder("key"),
+		Value: sarama.StringEncoder(userID),
+	}
+	keyProducer.ch.Input() <- &m
+}
+
+func InitNotifier(base *basecomponent.BaseDendrite) {
+	keyProducer.base = base
+	pro, _ := sarama.NewAsyncProducer(base.Cfg.Kafka.Addresses, nil)
+	keyProducer.ch = pro
+}

src/github.com/matrix-org/dendrite/encryptoapi/routing/routing.go

@@ -0,0 +1,71 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
+
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/encryptoapi/storage"
+	"github.com/matrix-org/util"
+)
+
+const pathPrefixR0 = "/_matrix/client/r0"
+const pathPrefixUnstable = "/_matrix/client/unstable"
+
+func Setup(
+	apiMux *mux.Router,
+	cfg config.Dendrite,
+	encryptionDB *storage.Database,
+	accountDB *accounts.Database,
+	deviceDB *devices.Database,
+) {
+	//r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter()
+	unstablemux := apiMux.PathPrefix(pathPrefixUnstable).Subrouter()
+
+	unstablemux.Handle("/keys/upload/{deviceID}",
+		common.MakeAuthAPI("upload keys", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			return UploadPKeys(req, encryptionDB, device.UserID, device.ID)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
+	unstablemux.Handle("/keys/upload",
+		common.MakeAuthAPI("upload keys", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			return UploadPKeys(req, encryptionDB, device.UserID, device.ID)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
+	unstablemux.Handle("/keys/query",
+		common.MakeAuthAPI("query keys", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			//vars := mux.Vars(req)
+			return QueryPKeys(req, encryptionDB, device.UserID, device.ID, deviceDB)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
+	unstablemux.Handle("/keys/claim",
+		common.MakeAuthAPI("claim keys", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			//vars := mux.Vars(req)
+			return ClaimOneTimeKeys(req, encryptionDB, device.UserID, device.ID, deviceDB)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
+
+}

src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_algorithm.go

@@ -0,0 +1,85 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"database/sql"
+	"context"
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/encryptoapi/types"
+)
+
+const algorithmSchema = `
+-- The media_repository table holds metadata for each media file stored and accessible to the local server,
+-- the actual file is stored separately.
+CREATE TABLE IF NOT EXISTS encrypt_algorithm (
+    device_id TEXT 				NOT NULL,
+    user_id TEXT 				NOT NULL,
+    algorithms TEXT 			NOT NULL
+);
+`
+
+const insertalSQL = `
+INSERT INTO encrypt_algorithm (device_id, user_id, algorithms) VALUES ($1, $2, $3)
+`
+
+const selectalSQL = `
+SELECT user_id, device_id, algorithms FROM encrypt_algorithm WHERE user_id = $1 AND device_id = $2
+`
+
+type alStatements struct {
+	insertAlStmt *sql.Stmt
+	selectAlStmt *sql.Stmt
+}
+
+func (s *alStatements) prepare(db *sql.DB) (err error) {
+	_, err = db.Exec(algorithmSchema)
+	if err != nil {
+		return
+	}
+	if s.insertAlStmt, err = db.Prepare(insertalSQL); err != nil {
+		return
+	}
+	if s.selectAlStmt, err = db.Prepare(selectalSQL); err != nil {
+		return
+	}
+	return
+}
+
+func (ks *alStatements) insertAl(
+	ctx context.Context, txn *sql.Tx,
+	userID, deviceID, algorithms string,
+) error {
+	stmt := common.TxStmt(txn, ks.insertAlStmt)
+	_, err := stmt.ExecContext(ctx, deviceID, userID, algorithms)
+	return err
+}
+
+func (ks *alStatements) selectAl(
+	ctx context.Context,
+	txn *sql.Tx,
+	userID, deviceID  string,
+) (holder types.AlHolder, err error) {
+
+	stmt := common.TxStmt(txn, ks.selectAlStmt)
+	row := stmt.QueryRowContext(ctx, userID, deviceID)
+	single := types.AlHolder{}
+	err = row.Scan(
+		&single.User_id,
+		&single.Device_id,
+		&single.Supported_algorithm,
+	)
+	return single, err
+}

src/github.com/matrix-org/dendrite/encryptoapi/storage/encrypt_keys_table.go

@@ -0,0 +1,192 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"database/sql"
+	"context"
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/encryptoapi/types"
+	"github.com/lib/pq"
+)
+
+const keysSchema = `
+-- The media_repository table holds metadata for each media file stored and accessible to the local server,
+-- the actual file is stored separately.
+CREATE TABLE IF NOT EXISTS encrypt_keys (
+    device_id TEXT 				NOT NULL,
+    user_id TEXT 				NOT NULL,
+    key_id TEXT 						,
+    key_type TEXT 				NOT NULL,
+    key_info TEXT 				NOT NULL,
+    algorithm TEXT 				NOT NULL,
+    signature TEXT 				NOT NULL
+);
+`
+
+const insertkeySQL = `
+INSERT INTO encrypt_keys (device_id, user_id, key_id, key_type, key_info, algorithm, signature)
+    VALUES ($1, $2, $3, $4, $5, $6, $7)
+`
+
+const selectkeySQL = `
+SELECT user_id, device_id, key_id, key_type, key_info, algorithm, signature FROM encrypt_keys WHERE user_id = $1 AND device_id = $2
+`
+
+const deleteSinglekeySQL = `
+SELECT user_id, device_id, key_id, key_type, key_info, algorithm, signature FROM encrypt_keys WHERE user_id = $1 AND device_id = $2 AND algorithm = $3
+`
+const selectSinglekeySQL = `
+DELETE FROM encrypt_keys WHERE user_id = $1 AND device_id = $2 AND algorithm = $3 AND key_id = $4
+`
+
+const selectInkeysSQL = `
+SELECT user_id, device_id, key_id, key_type, key_info, algorithm, signature FROM encrypt_keys WHERE user_id = $1 AND key_type = 'device_key' AND device_id = ANY($2)
+`
+
+const selectAllkeysSQL = `
+SELECT user_id, device_id, key_id, key_type, key_info, algorithm, signature FROM encrypt_keys WHERE user_id = $1 AND key_type = $2
+`
+
+type keyStatements struct {
+	insertKeyStmt       *sql.Stmt
+	selectKeyStmt       *sql.Stmt
+	selectInKeysStmt    *sql.Stmt
+	selectAllKeyStmt    *sql.Stmt
+	selectSingleKeyStmt *sql.Stmt
+	deleteSingleKeyStmt *sql.Stmt
+}
+
+func (s *keyStatements) prepare(db *sql.DB) (err error) {
+	_, err = db.Exec(keysSchema)
+	if err != nil {
+		return
+	}
+	if s.insertKeyStmt, err = db.Prepare(insertkeySQL); err != nil {
+		return
+	}
+	if s.selectKeyStmt, err = db.Prepare(selectkeySQL); err != nil {
+		return
+	}
+	if s.selectInKeysStmt, err = db.Prepare(selectInkeysSQL); err != nil {
+		return
+	}
+	if s.selectAllKeyStmt, err = db.Prepare(selectAllkeysSQL); err != nil {
+		return
+	}
+	if s.deleteSingleKeyStmt, err = db.Prepare(selectSinglekeySQL); err != nil {
+		return
+	}
+	if s.selectSingleKeyStmt, err = db.Prepare(deleteSinglekeySQL); err != nil {
+		return
+	}
+	return
+}
+
+func (ks *keyStatements) insertKey(
+	ctx context.Context, txn *sql.Tx,
+	deviceID, userID, keyID, keyTyp, keyInfo, algorithm, signature string,
+) error {
+	stmt := common.TxStmt(txn, ks.insertKeyStmt)
+	_, err := stmt.ExecContext(ctx, deviceID, userID, keyID, keyTyp, keyInfo, algorithm, signature)
+	return err
+}
+
+func (ks *keyStatements) selectKey(
+	ctx context.Context,
+	txn *sql.Tx,
+	deviceID, userID string,
+) (holders []types.KeyHolder, err error) {
+	stmt := common.TxStmt(txn, ks.selectKeyStmt)
+	rows, err := stmt.QueryContext(ctx, userID, deviceID)
+	if err != nil {
+		return nil, err
+	}
+	for rows.Next() {
+		single := &types.KeyHolder{}
+		if err := rows.Scan(
+			&single.User_id,
+			&single.Device_id,
+			&single.Key_id,
+			&single.Key_type,
+			&single.Key,
+			&single.Key_algorithm,
+			&single.Signature,
+		); err != nil {
+			return nil, err
+		}
+		holders = append(holders, *single)
+	}
+	return holders, err
+}
+func (ks *keyStatements) selectSingleKey(
+	ctx context.Context,
+	userID, deviceID, algorithm string,
+) (holder types.KeyHolder, err error) {
+	stmt := ks.selectSingleKeyStmt
+	row := stmt.QueryRowContext(ctx, userID, deviceID, algorithm)
+	if err != nil {
+		return holder, err
+	}
+	if err := row.Scan(
+		&holder.User_id,
+		&holder.Device_id,
+		&holder.Key_id,
+		&holder.Key_type,
+		&holder.Key,
+		&holder.Key_algorithm,
+		&holder.Signature,
+	); err != nil {
+		deleteStmt := ks.deleteSingleKeyStmt
+		deleteStmt.ExecContext(ctx, userID, deviceID, algorithm, holder.Key_id)
+		return holder, err
+	}
+	return holder, err
+}
+
+func (ks *keyStatements) selectInKeys(
+	ctx context.Context,
+	userID string,
+	arr []string,
+) (holders []types.KeyHolder, err error) {
+	rows := &sql.Rows{}
+	stmt := ks.selectAllKeyStmt
+	if len(arr) == 0 {
+		rows, err = stmt.QueryContext(ctx, userID, "device_key")
+	} else {
+		stmt = ks.selectInKeysStmt
+		list := pq.Array(arr)
+		rows, err = stmt.QueryContext(ctx, userID, list)
+	}
+	if err != nil {
+		return nil, err
+	}
+	for rows.Next() {
+		single := &types.KeyHolder{}
+		if err := rows.Scan(
+			&single.User_id,
+			&single.Device_id,
+			&single.Key_id,
+			&single.Key_type,
+			&single.Key,
+			&single.Key_algorithm,
+			&single.Signature,
+		); err != nil {
+			return nil, err
+		}
+		holders = append(holders, *single)
+	}
+	return holders, err
+}

src/github.com/matrix-org/dendrite/encryptoapi/storage/storage.go

@@ -0,0 +1,112 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"database/sql"
+	"github.com/matrix-org/dendrite/common"
+	"context"
+	"github.com/matrix-org/dendrite/encryptoapi/types"
+	"strings"
+)
+
+// Database represents a presence database.
+type Database struct {
+	db            *sql.DB
+	keyStatements keyStatements
+	alStatements  alStatements
+}
+
+// NewDatabase creates a new presence database
+func NewDatabase(dataSourceName string) (*Database, error) {
+	var db *sql.DB
+	var err error
+	if db, err = sql.Open("postgres_hook", dataSourceName); err != nil {
+		return nil, err
+	}
+	keyStatement := keyStatements{}
+	alStatement := alStatements{}
+	if err = keyStatement.prepare(db); err != nil {
+		return nil, err
+	}
+	if err = alStatement.prepare(db); err != nil {
+		return nil, err
+	}
+	return &Database{db: db, keyStatements: keyStatement, alStatements: alStatement}, nil
+}
+
+func (d *Database) InsertKey(
+	ctx context.Context,
+	deviceID, userID, keyID, keyTyp, keyInfo, al, sig string,
+) (err error) {
+	err = common.WithTransaction(d.db, func(txn *sql.Tx) error {
+		return d.keyStatements.insertKey(ctx, txn, deviceID, userID, keyID, keyTyp, keyInfo, al, sig)
+	})
+	return
+}
+
+func (d *Database) SelectOneTimeKeyCount(
+	ctx context.Context,
+	deviceID, userID string,
+) (m map[string]int, err error) {
+	m = make(map[string]int)
+	err = common.WithTransaction(d.db, func(txn *sql.Tx) error {
+		elems, err := d.keyStatements.selectKey(ctx, txn, deviceID, userID)
+		for _, val := range elems {
+			if _, ok := m[val.Key_algorithm]; !ok {
+				m[val.Key_algorithm] = 0
+			}
+			if val.Key_type == "one_time_key" {
+				m[val.Key_algorithm] += 1
+			}
+		}
+		return err
+	})
+	return
+}
+
+func (d *Database) QueryInRange(
+	ctx context.Context,
+	userID string,
+	arr []string,
+) (res []types.KeyHolder, err error) {
+	res, err = d.keyStatements.selectInKeys(ctx, userID, arr)
+	return
+}
+
+func (d *Database) InsertAl(ctx context.Context, uid, device string, al []string) (err error) {
+	err = common.WithTransaction(d.db, func(txn *sql.Tx) (err error) {
+		d.alStatements.insertAl(ctx, txn, uid, device, strings.Join(al, ","))
+		return
+	})
+	return
+}
+
+func (d *Database) SelectAl(ctx context.Context, uid, device string) (res []string, err error) {
+	err = common.WithTransaction(d.db, func(txn *sql.Tx) (err error) {
+		holder, err := d.alStatements.selectAl(ctx, txn, uid, device)
+		res = strings.Split(holder.Supported_algorithm, ",")
+		return
+	})
+	return
+}
+
+func (d *Database) SelectOneTimeKeySingle(
+	ctx context.Context,
+	userID, deviceID, algorithm string,
+) (holder types.KeyHolder, err error) {
+	holder, err = d.keyStatements.selectSingleKey(ctx, userID, deviceID, algorithm)
+	return
+}

src/github.com/matrix-org/dendrite/encryptoapi/types/claim.go

@@ -0,0 +1,54 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package types
+
+/*
+	{
+	  "timeout": 10000,
+	  "one_time_keys": {
+		"@alice:example.com": {
+		  "JLAFKJWSCS": "curve25519"
+		}
+	  }
+	}
+*/
+type ClaimRequest struct {
+	Timeout     int64                        `json:"timeout"`
+	ClaimDetail map[string]map[string]string `json:"one_time_keys"`
+}
+
+/*
+	{
+	  "failures": {},
+	  "one_time_keys": {
+		"@alice:example.com": {
+		  "JLAFKJWSCS": {
+			"signed_curve25519:AAAAHg": {
+			  "key": "zKbLg+NrIjpnagy+pIY6uPL4ZwEG2v+8F9lmgsnlZzs",
+			  "signatures": {
+				"@alice:example.com": {
+				  "ed25519:JLAFKJWSCS": "FLWxXqGbwrb8SM3Y795eB6OA8bwBcoMZFXBqnTn58AYWZSqiD45tlBVcDa2L7RwdKXebW/VzDlnfVJ+9jok1Bw"
+				}
+			  }
+			}
+		  }
+		}
+	  }
+	}
+*/
+type ClaimResponse struct {
+	Failures map[string]interface{} `json:"failures"`
+	ClaimBody map[string]map[string]map[string]interface{} `json:"one_time_keys"`
+}

src/github.com/matrix-org/dendrite/encryptoapi/types/query.go

@@ -0,0 +1,76 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package types
+
+/*
+	{
+	  "timeout": 10000,
+	  "device_keys": {
+		"@alice:example.com": ["DISYYYX","XYIISONM"]
+	  }
+	}
+*/
+type QueryRequest struct {
+	Timeout    int64               `json:"timeout"`
+	DeviceKeys map[string]interface{} `json:"device_keys"`
+	Token      string              `json:"token"`
+}
+
+/*
+	{
+	  "failures": {},
+	  "device_keys": {
+		"@alice:example.com": {
+		  "JLAFKJWSCS": {
+			"user_id": "@alice:example.com",
+			"device_id": "JLAFKJWSCS",
+			"algorithms": [
+			  "m.olm.curve25519-aes-sha256",
+			  "m.megolm.v1.aes-sha"
+			],
+			"keys": {
+			  "curve25519:JLAFKJWSCS": "3C5BFWi2Y8MaVvjM8M22DBmh24PmgR0nPvJOIArzgyI",
+			  "ed25519:JLAFKJWSCS": "lEuiRJBit0IG6nUf5pUzWTUEsRVVe/HJkoKuEww9ULI"
+			},
+			"signatures": {
+			  "@alice:example.com": {
+				"ed25519:JLAFKJWSCS": "dSO80A01XiigH3uBiDVx/EjzaoycHcjq9lfQX0uWsqxl2giMIiSPR8a4d291W1ihKJL/a+myXS367WT6NAIcBA"
+			  }
+			},
+			"unsigned": {
+			  "device_display_name": "Alice's mobile phone"
+			}
+		  }
+		}
+	  }
+	}
+*/
+type QueryResponse struct {
+	Failure    map[string]interface{}                `json:"failures"`
+	DeviceKeys map[string]map[string]DeviceKeysQuery `json:"device_keys"`
+}
+
+type DeviceKeysQuery struct {
+	UserId    string                       `json:"user_id"`
+	DeviceId  string                       `json:"device_id"`
+	Algorithm []string                     `json:"algorithms"`
+	Keys      map[string]string            `json:"keys"`
+	Signature map[string]map[string]string `json:"signatures"`
+	Unsigned  UnsignedDeviceInfo           `json:"unsigned"`
+}
+
+type UnsignedDeviceInfo struct {
+	Info string `json:"device_display_name"`
+}

src/github.com/matrix-org/dendrite/encryptoapi/types/storage.go

@@ -0,0 +1,30 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package types
+
+type KeyHolder struct {
+	User_id,
+	Device_id,
+	Signature, key,
+	Key_algorithm,
+	Key_id,
+	Key,
+	Key_type string
+}
+type AlHolder struct {
+	User_id,
+	Device_id,
+	Supported_algorithm string
+}

src/github.com/matrix-org/dendrite/encryptoapi/types/upload.go

@@ -0,0 +1,150 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package types
+
+/*
+	{
+	  "device_keys": {
+		"user_id": "@alice:example.com",
+		"device_id": "JLAFKJWSCS",
+		"algorithms": [
+		  "m.olm.curve25519-aes-sha256",
+		  "m.megolm.v1.aes-sha"
+		],
+		"keys": {
+		  "curve25519:JLAFKJWSCS": "3C5BFWi2Y8MaVvjM8M22DBmh24PmgR0nPvJOIArzgyI",
+		  "ed25519:JLAFKJWSCS": "lEuiRJBit0IG6nUf5pUzWTUEsRVVe/HJkoKuEww9ULI"
+		},
+		"signatures": {
+		  "@alice:example.com": {
+			"ed25519:JLAFKJWSCS": "dSO80A01XiigH3uBiDVx/EjzaoycHcjq9lfQX0uWsqxl2giMIiSPR8a4d291W1ihKJL/a+myXS367WT6NAIcBA"
+		  }
+		}
+	  },
+	  "one_time_keys": {
+		"curve25519:AAAAAQ": "/qyvZvwjiTxGdGU0RCguDCLeR+nmsb3FfNG3/Ve4vU8",
+		"signed_curve25519:AAAAHg": {
+		  "key": "zKbLg+NrIjpnagy+pIY6uPL4ZwEG2v+8F9lmgsnlZzs",
+		  "signatures": {
+			"@alice:example.com": {
+			  "ed25519:JLAFKJWSCS": "FLWxXqGbwrb8SM3Y795eB6OA8bwBcoMZFXBqnTn58AYWZSqiD45tlBVcDa2L7RwdKXebW/VzDlnfVJ+9jok1Bw"
+			}
+		  }
+		},
+		"signed_curve25519:AAAAHQ": {
+		  "key": "j3fR3HemM16M7CWhoI4Sk5ZsdmdfQHsKL1xuSft6MSw",
+		  "signatures": {
+			"@alice:example.com": {
+			  "ed25519:JLAFKJWSCS": "IQeCEPb9HFk217cU9kw9EOiusC6kMIkoIRnbnfOh5Oc63S1ghgyjShBGpu34blQomoalCyXWyhaaT3MrLZYQAA"
+			}
+		  }
+		}
+	  }
+	}
+*/
+type UploadEncrypt struct {
+	DeviceKeys DeviceKeys             `json:"device_keys"`
+	OneTimeKey map[string]interface{} `json:"one_time_keys"`
+}
+type UploadEncryptSpecific struct {
+	DeviceKeys DeviceKeys         `json:"device_keys"`
+	OneTimeKey OneTimeKeySpecific `json:"one_time_keys"`
+}
+
+/*
+	{
+	  "one_time_key_counts": {
+		"curve25519": 10,
+		"signed_curve25519": 20
+	  }
+	}
+*/
+type UploadResponse struct {
+	Count map[string]int `json:"one_time_key_counts"`
+}
+
+/*
+  "device_keys": {
+    "user_id": "@alice:example.com",
+    "device_id": "JLAFKJWSCS",
+    "algorithms": [
+      "m.olm.curve25519-aes-sha256",
+      "m.megolm.v1.aes-sha"
+    ],
+    "keys": {
+      "curve25519:JLAFKJWSCS": "3C5BFWi2Y8MaVvjM8M22DBmh24PmgR0nPvJOIArzgyI",
+      "ed25519:JLAFKJWSCS": "lEuiRJBit0IG6nUf5pUzWTUEsRVVe/HJkoKuEww9ULI"
+    },
+    "signatures": {
+      "@alice:example.com": {
+        "ed25519:JLAFKJWSCS": "dSO80A01XiigH3uBiDVx/EjzaoycHcjq9lfQX0uWsqxl2giMIiSPR8a4d291W1ihKJL/a+myXS367WT6NAIcBA"
+      }
+    }
+  }
+*/
+type DeviceKeys struct {
+	UserId    string                       `json:"user_id"`
+	DeviceId  string                       `json:"device_id"`
+	Algorithm []string                     `json:"algorithms"`
+	Keys      map[string]string            `json:"keys"`
+	Signature map[string]map[string]string `json:"signatures"`
+}
+
+/*
+    "signed_curve25519:AAAAHg": {
+      "key": "zKbLg+NrIjpnagy+pIY6uPL4ZwEG2v+8F9lmgsnlZzs",
+      "signatures": {
+        "@alice:example.com": {
+          "ed25519:JLAFKJWSCS": "FLWxXqGbwrb8SM3Y795eB6OA8bwBcoMZFXBqnTn58AYWZSqiD45tlBVcDa2L7RwdKXebW/VzDlnfVJ+9jok1Bw"
+        }
+      }
+	}
+*/
+type KeyObject struct {
+	Key       string                       `json:"key"`
+	Signature map[string]map[string]string `json:"signatures"`
+}
+
+/*
+    "one_time_keys": {
+		"curve25519:AAAAAQ": "/qyvZvwjiTxGdGU0RCguDCLeR+nmsb3FfNG3/Ve4vU8",
+		"signed_curve25519:AAAAHg": {
+		  "key": "zKbLg+NrIjpnagy+pIY6uPL4ZwEG2v+8F9lmgsnlZzs",
+		  "signatures": {
+			"@alice:example.com": {
+			  "ed25519:JLAFKJWSCS": "FLWxXqGbwrb8SM3Y795eB6OA8bwBcoMZFXBqnTn58AYWZSqiD45tlBVcDa2L7RwdKXebW/VzDlnfVJ+9jok1Bw"
+			}
+		  }
+		},
+		"signed_curve25519:AAAAHQ": {
+		  "key": "j3fR3HemM16M7CWhoI4Sk5ZsdmdfQHsKL1xuSft6MSw",
+		  "signatures": {
+			"@alice:example.com": {
+			  "ed25519:JLAFKJWSCS": "IQeCEPb9HFk217cU9kw9EOiusC6kMIkoIRnbnfOh5Oc63S1ghgyjShBGpu34blQomoalCyXWyhaaT3MrLZYQAA"
+			}
+		  }
+		}
+	}
+*/
+type OneTimeKey struct {
+	//KeyString map[string]string
+	//KeyObject map[string]KeyObject
+	KeySth map[string]interface{}
+}
+type OneTimeKeySpecific struct {
+	KeyString map[string]string
+	KeyObject map[string]KeyObject
+	//KeySth map[string]interface{}
+}