Aligning to master

cmd/dendrite-monolith-server/main.go

@@ -18,26 +18,20 @@ import (
 	"flag"
 	"net/http"
 
-<<<<<<< HEAD:cmd/dendrite-monolith-server/main.go
 	"github.com/matrix-org/dendrite/appservice"
-=======
 	"github.com/matrix-org/dendrite/common/keydb"
 	"github.com/matrix-org/dendrite/common/transactions"
 	"github.com/matrix-org/dendrite/typingserver"
 
->>>>>>> 8b4b3c6fc46900e9bfe5e234eda309200662b34a:src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go
 	"github.com/matrix-org/dendrite/clientapi"
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/common/basecomponent"
-	"github.com/matrix-org/dendrite/common/keydb"
-	"github.com/matrix-org/dendrite/common/transactions"
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/federationsender"
 	"github.com/matrix-org/dendrite/mediaapi"
 	"github.com/matrix-org/dendrite/publicroomsapi"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/syncapi"
-	"github.com/matrix-org/dendrite/typingserver"
 	"github.com/matrix-org/dendrite/typingserver/cache"
 
 	"github.com/matrix-org/dendrite/encryptoapi"
@@ -80,12 +74,9 @@ func main() {
 	federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery)
 	mediaapi.SetupMediaAPIComponent(base, deviceDB)
 	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB)
-<<<<<<< HEAD:cmd/dendrite-monolith-server/main.go
+	// syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query)
-	syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query)
-=======
 	syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, encryptDB)
 	//appservice.SetupAppServiceAPIComponent(base, accountDB, deviceDB, federation, alias, query, transactions.New())
->>>>>>> 8b4b3c6fc46900e9bfe5e234eda309200662b34a:src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go
 
 	httpHandler := common.WrapHandlerInCORS(base.APIMux)
 

src/github.com/matrix-org/dendrite/syncapi/routing/std.go

@@ -1,89 +0,0 @@
-package routing
-
-import (
-	"encoding/json"
-	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
-	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/syncapi/storage"
-	"github.com/matrix-org/dendrite/syncapi/sync"
-	"github.com/matrix-org/dendrite/syncapi/types"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-	"net/http"
-)
-
-// SendToDevice this is a function for calling process of send-to-device messages those bypassed DAG
-func SendToDevice(
-	req *http.Request,
-	sender string,
-	syncDB *storage.SyncServerDatabase,
-	deviceDB *devices.Database,
-	eventType, txnID string,
-	notifier *sync.Notifier,
-) util.JSONResponse {
-	ctx := req.Context()
-	stdRq := types.StdRequest{}
-	httputil.UnmarshalJSONRequest(req, &stdRq)
-	for uid, deviceMap := range stdRq.Sender {
-
-		// federation consideration todo:
-		// if uid is remote domain a fed process should go
-		if false {
-			// federation process
-			return util.JSONResponse{}
-		}
-
-		// uid is local domain
-		for device, cont := range deviceMap {
-			jsonBuffer, err := json.Marshal(cont)
-			if err != nil {
-				return util.JSONResponse{
-					Code: http.StatusForbidden,
-					JSON: struct{}{},
-				}
-			}
-			ev := types.StdHolder{
-				Sender:   sender,
-				Event:    jsonBuffer,
-				EventTyp: eventType,
-			}
-			var pos int64
-
-			// wildcard all devices
-			if device == "*" {
-				var deviceCollection []authtypes.Device
-				var localpart string
-				localpart, _, _ = gomatrixserverlib.SplitID('@', uid)
-				deviceCollection, err = deviceDB.GetDevicesByLocalpart(ctx, localpart)
-				for _, val := range deviceCollection {
-					pos, err = syncDB.InsertStdMessage(ctx, ev, txnID, uid, val.ID)
-					notifier.OnNewEvent(nil, uid, types.StreamPosition(pos))
-				}
-				if err != nil {
-					return util.JSONResponse{
-						Code: http.StatusForbidden,
-						JSON: struct{}{},
-					}
-				}
-				return util.JSONResponse{
-					Code: http.StatusOK,
-					JSON: struct{}{},
-				}
-			}
-			pos, err = syncDB.InsertStdMessage(ctx, ev, txnID, uid, device)
-			if err != nil {
-				return util.JSONResponse{
-					Code: http.StatusForbidden,
-					JSON: struct{}{},
-				}
-			}
-			notifier.OnNewEvent(nil, uid, types.StreamPosition(pos))
-		}
-	}
-
-	return util.JSONResponse{
-		Code: http.StatusOK,
-		JSON: struct{}{},
-	}
-}

syncapi/routing/routing.go

@@ -32,17 +32,13 @@ const pathPrefixR0 = "/_matrix/client/r0"
 const pathPrefixUnstable = "/_matrix/client/unstable"
 
 // Setup configures the given mux with sync-server listeners
-<<<<<<< HEAD:syncapi/routing/routing.go
 //
 // Due to Setup being used to call many other functions, a gocyclo nolint is
 // applied:
 // nolint: gocyclo
-func Setup(apiMux *mux.Router, srp *sync.RequestPool, syncDB *storage.SyncServerDatasource, deviceDB *devices.Database) {
+func Setup(apiMux *mux.Router, srp *sync.RequestPool, syncDB *storage.SyncServerDatasource, deviceDB *devices.Database, notifier *sync.Notifier, encryptDB *encryptoapi.Database) {
-=======
-func Setup(apiMux *mux.Router, srp *sync.RequestPool, syncDB *storage.SyncServerDatabase, deviceDB *devices.Database, notifier *sync.Notifier, encryptDB *encryptoapi.Database) {
->>>>>>> 8b4b3c6fc46900e9bfe5e234eda309200662b34a:src/github.com/matrix-org/dendrite/syncapi/routing/routing.go
 	r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter()
-	unstablemux := apiMux.PathPrefix(pathPrefixUnstable).Subrouter()
+	// unstablemux := apiMux.PathPrefix(pathPrefixUnstable).Subrouter()
 
 	authData := auth.Data{
 		AccountDB:   nil,
@@ -79,12 +75,12 @@ func Setup(apiMux *mux.Router, srp *sync.RequestPool, syncDB *storage.SyncServer
 		return OnIncomingStateTypeRequest(req, syncDB, vars["roomID"], vars["type"], vars["stateKey"])
 	})).Methods(http.MethodGet, http.MethodOptions)
 
-	unstablemux.Handle("/sendToDevice/{eventType}/{txnId}",
+	// unstablemux.Handle("/sendToDevice/{eventType}/{txnId}",
-		common.MakeAuthAPI("look up changes", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+	// 	common.MakeAuthAPI("look up changes", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+	// 		vars := mux.Vars(req)
-			eventType := vars["eventType"]
+	// 		eventType := vars["eventType"]
-			txnID := vars["txnId"]
+	// 		txnID := vars["txnId"]
-			return SendToDevice(req, device.UserID, syncDB, deviceDB, eventType, txnID, notifier)
+	// 		return SendToDevice(req, device.UserID, syncDB, deviceDB, eventType, txnID, notifier)
-		}),
+	// 	}),
-	).Methods(http.MethodPut, http.MethodOptions)
+	// ).Methods(http.MethodPut, http.MethodOptions)
 }

syncapi/routing/std.go

@@ -0,0 +1,89 @@
+package routing
+
+// import (
+// 	"encoding/json"
+// 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+// 	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
+// 	"github.com/matrix-org/dendrite/clientapi/httputil"
+// 	"github.com/matrix-org/dendrite/syncapi/storage"
+// 	"github.com/matrix-org/dendrite/syncapi/sync"
+// 	"github.com/matrix-org/dendrite/syncapi/types"
+// 	"github.com/matrix-org/gomatrixserverlib"
+// 	"github.com/matrix-org/util"
+// 	"net/http"
+// )
+
+// // SendToDevice this is a function for calling process of send-to-device messages those bypassed DAG
+// func SendToDevice(
+// 	req *http.Request,
+// 	sender string,
+// 	syncDB *storage.SyncServerDatasource,
+// 	deviceDB *devices.Database,
+// 	eventType, txnID string,
+// 	notifier *sync.Notifier,
+// ) util.JSONResponse {
+// 	ctx := req.Context()
+// 	stdRq := types.StdRequest{}
+// 	httputil.UnmarshalJSONRequest(req, &stdRq)
+// 	for uid, deviceMap := range stdRq.Sender {
+
+// 		// federation consideration todo:
+// 		// if uid is remote domain a fed process should go
+// 		if false {
+// 			// federation process
+// 			return util.JSONResponse{}
+// 		}
+
+// 		// uid is local domain
+// 		for device, cont := range deviceMap {
+// 			jsonBuffer, err := json.Marshal(cont)
+// 			if err != nil {
+// 				return util.JSONResponse{
+// 					Code: http.StatusForbidden,
+// 					JSON: struct{}{},
+// 				}
+// 			}
+// 			ev := types.StdHolder{
+// 				Sender:   sender,
+// 				Event:    jsonBuffer,
+// 				EventTyp: eventType,
+// 			}
+// 			var pos int64
+
+// 			// wildcard all devices
+// 			if device == "*" {
+// 				var deviceCollection []authtypes.Device
+// 				var localpart string
+// 				localpart, _, _ = gomatrixserverlib.SplitID('@', uid)
+// 				deviceCollection, err = deviceDB.GetDevicesByLocalpart(ctx, localpart)
+// 				for _, val := range deviceCollection {
+// 					pos, err = syncDB.InsertStdMessage(ctx, ev, txnID, uid, val.ID)
+// 					notifier.OnNewEvent(nil, uid, types.StreamPosition(pos))
+// 				}
+// 				if err != nil {
+// 					return util.JSONResponse{
+// 						Code: http.StatusForbidden,
+// 						JSON: struct{}{},
+// 					}
+// 				}
+// 				return util.JSONResponse{
+// 					Code: http.StatusOK,
+// 					JSON: struct{}{},
+// 				}
+// 			}
+// 			pos, err = syncDB.InsertStdMessage(ctx, ev, txnID, uid, device)
+// 			if err != nil {
+// 				return util.JSONResponse{
+// 					Code: http.StatusForbidden,
+// 					JSON: struct{}{},
+// 				}
+// 			}
+// 			notifier.OnNewEvent(nil, uid, types.StreamPosition(pos))
+// 		}
+// 	}
+
+// 	return util.JSONResponse{
+// 		Code: http.StatusOK,
+// 		JSON: struct{}{},
+// 	}
+// }

syncapi/storage/syncserver.go

@@ -27,12 +27,8 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/roomserver/api"
 
-<<<<<<< HEAD:syncapi/storage/syncserver.go
 	// Import the postgres database driver.
 	_ "github.com/lib/pq"
-=======
-	"encoding/json"
->>>>>>> 8b4b3c6fc46900e9bfe5e234eda309200662b34a:src/github.com/matrix-org/dendrite/syncapi/storage/syncserver.go
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/dendrite/typingserver/cache"
@@ -55,7 +51,7 @@ type streamEvent struct {
 	transactionID  *api.TransactionID
 }
 
-// SyncServerDatabase represents a sync server datasource which manages
+// SyncServerDatasource represents a sync server datasource which manages
 // both the database for PDUs and caches for EDUs.
 type SyncServerDatasource struct {
 	db *sql.DB
@@ -64,14 +60,11 @@ type SyncServerDatasource struct {
 	events      outputRoomEventsStatements
 	roomstate   currentRoomStateStatements
 	invites     inviteEventsStatements
-<<<<<<< HEAD:syncapi/storage/syncserver.go
 	typingCache *cache.TypingCache
-=======
 	stdMsg      stdEventsStatements
->>>>>>> 8b4b3c6fc46900e9bfe5e234eda309200662b34a:src/github.com/matrix-org/dendrite/syncapi/storage/syncserver.go
 }
 
-// NewSyncServerDatabase creates a new sync server database
+// NewSyncServerDatasource creates a new sync server database
 func NewSyncServerDatasource(dbDataSourceName string) (*SyncServerDatasource, error) {
 	var d SyncServerDatasource
 	var err error
@@ -93,13 +86,10 @@ func NewSyncServerDatasource(dbDataSourceName string) (*SyncServerDatasource, er
 	if err := d.invites.prepare(d.db); err != nil {
 		return nil, err
 	}
-<<<<<<< HEAD:syncapi/storage/syncserver.go
 	d.typingCache = cache.NewTypingCache()
-=======
 	if err := d.stdMsg.prepare(d.db); err != nil {
 		return nil, err
 	}
->>>>>>> 8b4b3c6fc46900e9bfe5e234eda309200662b34a:src/github.com/matrix-org/dendrite/syncapi/storage/syncserver.go
 	return &d, nil
 }
 
@@ -235,22 +225,22 @@ func (d *SyncServerDatasource) syncPositionTx(
 	if maxInviteID > maxEventID {
 		maxEventID = maxInviteID
 	}
-<<<<<<< HEAD:syncapi/storage/syncserver.go
+
+	// E2EE changes look
+	// maxStdID, err := d.stdMsg.selectMaxStdID(ctx, txn)
+	// if err != nil {
+	// 	return 0, err
+	// }
+	// if maxStdID > maxID {
+	// 	maxID = maxStdID
+	// }
+	// return types.StreamPosition(maxID), nil
 	sp.PDUPosition = maxEventID
 
 	sp.TypingPosition = d.typingCache.GetLatestSyncPosition()
 
 	return
-=======
+
-	maxStdID, err := d.stdMsg.selectMaxStdID(ctx, txn)
-	if err != nil {
-		return 0, err
-	}
-	if maxStdID > maxID {
-		maxID = maxStdID
-	}
-	return types.StreamPosition(maxID), nil
->>>>>>> 8b4b3c6fc46900e9bfe5e234eda309200662b34a:src/github.com/matrix-org/dendrite/syncapi/storage/syncserver.go
 }
 
 // addPDUDeltaToResponse adds all PDU deltas to a sync response.
@@ -977,7 +967,7 @@ del / maxID / select in range / insert
 */
 
 // DelStdMessage delete message for a given maxID, those below would be deleted
-func (d *SyncServerDatabase) DelStdMessage(
+func (d *SyncServerDatasource) DelStdMessage(
 	ctx context.Context, targetUID, targetDevice string, maxID int64,
 ) (err error) {
 	err = common.WithTransaction(d.db, func(txn *sql.Tx) error {
@@ -988,7 +978,7 @@ func (d *SyncServerDatabase) DelStdMessage(
 }
 
 // InsertStdMessage insert std message
-func (d *SyncServerDatabase) InsertStdMessage(
+func (d *SyncServerDatasource) InsertStdMessage(
 	ctx context.Context, stdEvent types.StdHolder, transactionID, targetUID, targetDevice string,
 ) (pos int64, err error) {
 	err = common.WithTransaction(d.db, func(txn *sql.Tx) error {
@@ -1000,7 +990,7 @@ func (d *SyncServerDatabase) InsertStdMessage(
 }
 
 // SelectMaxStdID select maximum id in std stream
-func (d *SyncServerDatabase) SelectMaxStdID(
+func (d *SyncServerDatasource) SelectMaxStdID(
 	ctx context.Context,
 ) (maxID int64, err error) {
 	err = common.WithTransaction(d.db, func(txn *sql.Tx) error {
@@ -1012,7 +1002,7 @@ func (d *SyncServerDatabase) SelectMaxStdID(
 }
 
 // SelectRangedStd select a range of std messages
-func (d *SyncServerDatabase) SelectRangedStd(
+func (d *SyncServerDatasource) SelectRangedStd(
 	ctx context.Context,
 	targetUserID, targetDeviceID string,
 	endPos int64,
@@ -1028,7 +1018,7 @@ func (d *SyncServerDatabase) SelectRangedStd(
 // StdEXT : send to device extension process
 func StdEXT(
 	ctx context.Context,
-	syncDB *SyncServerDatabase,
+	syncDB *SyncServerDatasource,
 	respIn types.Response,
 	userID, deviceID string,
 	since int64,

syncapi/sync/requestpool.go

@@ -45,13 +45,8 @@ func NewRequestPool(db *storage.SyncServerDatasource, n *Notifier, adb *accounts
 // OnIncomingSyncRequest is called when a client makes a /sync request. This function MUST be
 // called in a dedicated goroutine for this request. This function will block the goroutine
 // until a response is ready, or it times out.
-<<<<<<< HEAD:syncapi/sync/requestpool.go
-func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtypes.Device) util.JSONResponse {
-	var syncData *types.Response
-
-=======
 func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtypes.Device, encryptDB *encryptoapi.Database) util.JSONResponse {
->>>>>>> 8b4b3c6fc46900e9bfe5e234eda309200662b34a:src/github.com/matrix-org/dendrite/syncapi/sync/requestpool.go
+	var syncData *types.Response	
 	// Extract values from request
 	logger := util.GetLogger(req.Context())
 	userID := device.UserID
@@ -120,16 +115,14 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype
 		// of calculating the sync only to get timed out before we
 		// can respond
 
-<<<<<<< HEAD:syncapi/sync/requestpool.go
+		// syncData, err = rp.currentSyncForUser(*syncReq, currPos)
-		syncData, err = rp.currentSyncForUser(*syncReq, currPos)
-=======
 		syncData, err := rp.currentSyncForUser(*syncReq, currPos)
 
 		// std extension consideration
-		syncData = storage.StdEXT(syncReq.ctx, rp.db, *syncData, syncReq.device.UserID, syncReq.device.ID, int64(currPos))
+		// Have to check which "position" must be used here
+		// syncData = storage.StdEXT(syncReq.ctx, rp.db, *syncData, syncReq.device.UserID, syncReq.device.ID, int64(currPos))
 		syncData = KeyCountEXT(syncReq.ctx, encryptDB, *syncData, syncReq.device.UserID, syncReq.device.ID)
 
->>>>>>> 8b4b3c6fc46900e9bfe5e234eda309200662b34a:src/github.com/matrix-org/dendrite/syncapi/sync/requestpool.go
 		if err != nil {
 			return httputil.LogThenError(req, err)
 		}

syncapi/syncapi.go

@@ -72,7 +72,6 @@ func SetupSyncAPIComponent(
 		logrus.WithError(err).Panicf("failed to start client data consumer")
 	}
 
-<<<<<<< HEAD:syncapi/syncapi.go
 	typingConsumer := consumers.NewOutputTypingEventConsumer(
 		base.Cfg, base.KafkaConsumer, notifier, syncDB,
 	)
@@ -80,8 +79,6 @@ func SetupSyncAPIComponent(
 		logrus.WithError(err).Panicf("failed to start typing server consumer")
 	}
 
-	routing.Setup(base.APIMux, requestPool, syncDB, deviceDB)
+	// routing.Setup(base.APIMux, requestPool, syncDB, deviceDB)
-=======
 	routing.Setup(base.APIMux, requestPool, syncDB, deviceDB, notifier, encryptDB)
->>>>>>> 8b4b3c6fc46900e9bfe5e234eda309200662b34a:src/github.com/matrix-org/dendrite/syncapi/syncapi.go
 }