diff --git a/appservice/appservice.go b/appservice/appservice.go index 181799879..e52db2c28 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -44,12 +44,11 @@ func SetupAppServiceAPIComponent( accountsDB accounts.Database, deviceDB devices.Database, federation *gomatrixserverlib.FederationClient, - roomserverAliasAPI roomserverAPI.RoomserverAliasAPI, - roomserverQueryAPI roomserverAPI.RoomserverQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, transactionsCache *transactions.Cache, ) appserviceAPI.AppServiceQueryAPI { // Create a connection to the appservice postgres DB - appserviceDB, err := storage.NewDatabase(string(base.Cfg.Database.AppService)) + appserviceDB, err := storage.NewDatabase(string(base.Cfg.Database.AppService), base.Cfg.DbProperties()) if err != nil { logrus.WithError(err).Panicf("failed to connect to appservice db") } @@ -87,7 +86,7 @@ func SetupAppServiceAPIComponent( consumer := consumers.NewOutputRoomEventConsumer( base.Cfg, base.KafkaConsumer, accountsDB, appserviceDB, - roomserverQueryAPI, roomserverAliasAPI, workerStates, + rsAPI, workerStates, ) if err := consumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start appservice roomserver consumer") @@ -100,7 +99,7 @@ func SetupAppServiceAPIComponent( // Set up HTTP Endpoints routing.Setup( - base.APIMux, base.Cfg, roomserverQueryAPI, roomserverAliasAPI, + base.APIMux, base.Cfg, rsAPI, accountsDB, federation, transactionsCache, ) diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index 6ae58e85c..b7f689249 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -26,8 +26,8 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/Shopify/sarama" log "github.com/sirupsen/logrus" - sarama "gopkg.in/Shopify/sarama.v1" ) // OutputRoomEventConsumer consumes events that originated in the room server. @@ -35,8 +35,7 @@ type OutputRoomEventConsumer struct { roomServerConsumer *common.ContinualConsumer db accounts.Database asDB storage.Database - query api.RoomserverQueryAPI - alias api.RoomserverAliasAPI + rsAPI api.RoomserverInternalAPI serverName string workerStates []types.ApplicationServiceWorkerState } @@ -48,8 +47,7 @@ func NewOutputRoomEventConsumer( kafkaConsumer sarama.Consumer, store accounts.Database, appserviceDB storage.Database, - queryAPI api.RoomserverQueryAPI, - aliasAPI api.RoomserverAliasAPI, + rsAPI api.RoomserverInternalAPI, workerStates []types.ApplicationServiceWorkerState, ) *OutputRoomEventConsumer { consumer := common.ContinualConsumer{ @@ -61,8 +59,7 @@ func NewOutputRoomEventConsumer( roomServerConsumer: &consumer, db: store, asDB: appserviceDB, - query: queryAPI, - alias: aliasAPI, + rsAPI: rsAPI, serverName: string(cfg.Matrix.ServerName), workerStates: workerStates, } @@ -139,7 +136,7 @@ func (s *OutputRoomEventConsumer) lookupMissingStateEvents( // Request the missing events from the roomserver eventReq := api.QueryEventsByIDRequest{EventIDs: missing} var eventResp api.QueryEventsByIDResponse - if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { + if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { return nil, err } @@ -200,7 +197,7 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont // Check all known room aliases of the room the event came from queryReq := api.GetAliasesForRoomIDRequest{RoomID: event.RoomID()} var queryRes api.GetAliasesForRoomIDResponse - if err := s.alias.GetAliasesForRoomID(ctx, &queryReq, &queryRes); err == nil { + if err := s.rsAPI.GetAliasesForRoomID(ctx, &queryReq, &queryRes); err == nil { for _, alias := range queryRes.Aliases { if appservice.IsInterestedInRoomAlias(alias) { return true diff --git a/appservice/routing/routing.go b/appservice/routing/routing.go index 42fa80520..9f59e05f7 100644 --- a/appservice/routing/routing.go +++ b/appservice/routing/routing.go @@ -37,7 +37,7 @@ const pathPrefixApp = "/_matrix/app/v1" // nolint: gocyclo func Setup( apiMux *mux.Router, cfg *config.Dendrite, // nolint: unparam - queryAPI api.RoomserverQueryAPI, aliasAPI api.RoomserverAliasAPI, // nolint: unparam + rsAPI api.RoomserverInternalAPI, // nolint: unparam accountDB accounts.Database, // nolint: unparam federation *gomatrixserverlib.FederationClient, // nolint: unparam transactionsCache *transactions.Cache, // nolint: unparam diff --git a/appservice/storage/postgres/storage.go b/appservice/storage/postgres/storage.go index e145eeee2..475db6fc2 100644 --- a/appservice/storage/postgres/storage.go +++ b/appservice/storage/postgres/storage.go @@ -21,6 +21,7 @@ import ( // Import postgres database driver _ "github.com/lib/pq" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -33,10 +34,10 @@ type Database struct { } // NewDatabase opens a new database -func NewDatabase(dataSourceName string) (*Database, error) { +func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (*Database, error) { var result Database var err error - if result.db, err = sqlutil.Open("postgres", dataSourceName); err != nil { + if result.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/appservice/storage/sqlite3/storage.go b/appservice/storage/sqlite3/storage.go index 0cd1e4abc..0b0590f6d 100644 --- a/appservice/storage/sqlite3/storage.go +++ b/appservice/storage/sqlite3/storage.go @@ -37,7 +37,7 @@ type Database struct { func NewDatabase(dataSourceName string) (*Database, error) { var result Database var err error - if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/appservice/storage/storage.go b/appservice/storage/storage.go index 9fbd2a1f3..bf0a9b0c8 100644 --- a/appservice/storage/storage.go +++ b/appservice/storage/storage.go @@ -21,19 +21,22 @@ import ( "github.com/matrix-org/dendrite/appservice/storage/postgres" "github.com/matrix-org/dendrite/appservice/storage/sqlite3" + "github.com/matrix-org/dendrite/common" ) -func NewDatabase(dataSourceName string) (Database, error) { +// NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) +// and sets DB connection parameters +func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.NewDatabase(dataSourceName) + return postgres.NewDatabase(dataSourceName, dbProperties) } switch uri.Scheme { case "postgres": - return postgres.NewDatabase(dataSourceName) + return postgres.NewDatabase(dataSourceName, dbProperties) case "file": return sqlite3.NewDatabase(dataSourceName) default: - return postgres.NewDatabase(dataSourceName) + return postgres.NewDatabase(dataSourceName, dbProperties) } } diff --git a/appservice/storage/storage_wasm.go b/appservice/storage/storage_wasm.go index 2bd1433f9..ff0330ae0 100644 --- a/appservice/storage/storage_wasm.go +++ b/appservice/storage/storage_wasm.go @@ -19,9 +19,13 @@ import ( "net/url" "github.com/matrix-org/dendrite/appservice/storage/sqlite3" + "github.com/matrix-org/dendrite/common" ) -func NewDatabase(dataSourceName string) (Database, error) { +func NewDatabase( + dataSourceName string, + dbProperties common.DbProperties, // nolint:unparam +) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return nil, fmt.Errorf("Cannot use postgres implementation") diff --git a/are-we-synapse-yet.list b/are-we-synapse-yet.list index 5a900b3ef..71b05d1c4 100644 --- a/are-we-synapse-yet.list +++ b/are-we-synapse-yet.list @@ -28,6 +28,10 @@ log POST /login returns the same device_id as that in the request log POST /login can log in as a user with just the local part of the id log POST /login as non-existing user is rejected log POST /login wrong password is rejected +log Interactive authentication types include SSO +log Can perform interactive authentication with SSO +log The user must be consistent through an interactive authentication session with SSO +log The operation must be consistent through an interactive authentication session v1s GET /events initially v1s GET /initialSync initially csa Version responds 200 OK with valid structure @@ -44,6 +48,7 @@ dev DELETE /device/{deviceId} dev DELETE /device/{deviceId} requires UI auth user to match device owner dev DELETE /device/{deviceId} with no body gives a 401 dev The deleted device must be consistent through an interactive auth session +dev Users receive device_list updates for their own devices pre GET /presence/:user_id/status fetches initial status pre PUT /presence/:user_id/status updates my presence crm POST /createRoom makes a public room @@ -827,4 +832,4 @@ syn Multiple calls to /sync should not cause 500 errors gst Guest user can call /events on another world_readable room (SYN-606) gst Real user can call /events on another world_readable room (SYN-606) gst Events come down the correct room -pub Asking for a remote rooms list, but supplying the local server's name, returns the local rooms list \ No newline at end of file +pub Asking for a remote rooms list, but supplying the local server's name, returns the local rooms list diff --git a/build.sh b/build.sh index 3ef148891..087f4ae72 100755 --- a/build.sh +++ b/build.sh @@ -3,4 +3,6 @@ # Put installed packages into ./bin export GOBIN=$PWD/`dirname $0`/bin -go install -v $PWD/`dirname $0`/cmd/... \ No newline at end of file +go install -v $PWD/`dirname $0`/cmd/... + +GOOS=js GOARCH=wasm go build -o main.wasm ./cmd/dendritejs diff --git a/clientapi/auth/storage/accounts/postgres/storage.go b/clientapi/auth/storage/accounts/postgres/storage.go index 8ce367a3e..7e2073ec0 100644 --- a/clientapi/auth/storage/accounts/postgres/storage.go +++ b/clientapi/auth/storage/accounts/postgres/storage.go @@ -44,10 +44,10 @@ type Database struct { } // NewDatabase creates a new accounts and profiles database -func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { +func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sqlutil.Open("postgres", dataSourceName); err != nil { + if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } partitions := common.PartitionOffsetStatements{} diff --git a/clientapi/auth/storage/accounts/sqlite3/storage.go b/clientapi/auth/storage/accounts/sqlite3/storage.go index e190ba6c2..30a93e7ed 100644 --- a/clientapi/auth/storage/accounts/sqlite3/storage.go +++ b/clientapi/auth/storage/accounts/sqlite3/storage.go @@ -50,7 +50,7 @@ type Database struct { func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil { return nil, err } partitions := common.PartitionOffsetStatements{} diff --git a/clientapi/auth/storage/accounts/storage.go b/clientapi/auth/storage/accounts/storage.go index c643a4d0a..394cc5e16 100644 --- a/clientapi/auth/storage/accounts/storage.go +++ b/clientapi/auth/storage/accounts/storage.go @@ -21,20 +21,23 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/postgres" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" ) -func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) { +// NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) +// and sets postgres connection parameters +func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.NewDatabase(dataSourceName, serverName) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName) } switch uri.Scheme { case "postgres": - return postgres.NewDatabase(dataSourceName, serverName) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName) case "file": return sqlite3.NewDatabase(dataSourceName, serverName) default: - return postgres.NewDatabase(dataSourceName, serverName) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName) } } diff --git a/clientapi/auth/storage/accounts/storage_wasm.go b/clientapi/auth/storage/accounts/storage_wasm.go index 828afc6b4..61f9d6997 100644 --- a/clientapi/auth/storage/accounts/storage_wasm.go +++ b/clientapi/auth/storage/accounts/storage_wasm.go @@ -19,10 +19,15 @@ import ( "net/url" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" ) -func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) { +func NewDatabase( + dataSourceName string, + dbProperties common.DbProperties, // nolint:unparam + serverName gomatrixserverlib.ServerName, +) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return nil, fmt.Errorf("Cannot use postgres implementation") diff --git a/clientapi/auth/storage/devices/postgres/storage.go b/clientapi/auth/storage/devices/postgres/storage.go index 3f613cf32..57c268df8 100644 --- a/clientapi/auth/storage/devices/postgres/storage.go +++ b/clientapi/auth/storage/devices/postgres/storage.go @@ -36,10 +36,10 @@ type Database struct { } // NewDatabase creates a new device database -func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { +func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sqlutil.Open("postgres", dataSourceName); err != nil { + if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } d := devicesStatements{} diff --git a/clientapi/auth/storage/devices/sqlite3/storage.go b/clientapi/auth/storage/devices/sqlite3/storage.go index 85a8def2c..b69d6278c 100644 --- a/clientapi/auth/storage/devices/sqlite3/storage.go +++ b/clientapi/auth/storage/devices/sqlite3/storage.go @@ -41,7 +41,7 @@ type Database struct { func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil { return nil, err } d := devicesStatements{} diff --git a/clientapi/auth/storage/devices/storage.go b/clientapi/auth/storage/devices/storage.go index 99211db28..ec47a3272 100644 --- a/clientapi/auth/storage/devices/storage.go +++ b/clientapi/auth/storage/devices/storage.go @@ -21,20 +21,23 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/postgres" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" ) -func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) { +// NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) +// and sets postgres connection parameters +func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.NewDatabase(dataSourceName, serverName) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName) } switch uri.Scheme { case "postgres": - return postgres.NewDatabase(dataSourceName, serverName) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName) case "file": return sqlite3.NewDatabase(dataSourceName, serverName) default: - return postgres.NewDatabase(dataSourceName, serverName) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName) } } diff --git a/clientapi/auth/storage/devices/storage_wasm.go b/clientapi/auth/storage/devices/storage_wasm.go index 322852888..e25b7c640 100644 --- a/clientapi/auth/storage/devices/storage_wasm.go +++ b/clientapi/auth/storage/devices/storage_wasm.go @@ -19,10 +19,15 @@ import ( "net/url" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" ) -func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) { +func NewDatabase( + dataSourceName string, + dbProperties common.DbProperties, // nolint:unparam + serverName gomatrixserverlib.ServerName, +) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return nil, fmt.Errorf("Cannot use postgres implementation") diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 1339f7c8c..f81e0242c 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -38,15 +38,13 @@ func SetupClientAPIComponent( accountsDB accounts.Database, federation *gomatrixserverlib.FederationClient, keyRing *gomatrixserverlib.KeyRing, - aliasAPI roomserverAPI.RoomserverAliasAPI, - inputAPI roomserverAPI.RoomserverInputAPI, - queryAPI roomserverAPI.RoomserverQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, eduInputAPI eduServerAPI.EDUServerInputAPI, asAPI appserviceAPI.AppServiceQueryAPI, transactionsCache *transactions.Cache, - fedSenderAPI federationSenderAPI.FederationSenderQueryAPI, + fsAPI federationSenderAPI.FederationSenderInternalAPI, ) { - roomserverProducer := producers.NewRoomserverProducer(inputAPI, queryAPI) + roomserverProducer := producers.NewRoomserverProducer(rsAPI) eduProducer := producers.NewEDUServerProducer(eduInputAPI) userUpdateProducer := &producers.UserUpdateProducer{ @@ -60,15 +58,15 @@ func SetupClientAPIComponent( } consumer := consumers.NewOutputRoomEventConsumer( - base.Cfg, base.KafkaConsumer, accountsDB, queryAPI, + base.Cfg, base.KafkaConsumer, accountsDB, rsAPI, ) if err := consumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start room server consumer") } routing.Setup( - base.APIMux, base.Cfg, roomserverProducer, queryAPI, aliasAPI, asAPI, + base.APIMux, base.Cfg, roomserverProducer, rsAPI, asAPI, accountsDB, deviceDB, federation, *keyRing, userUpdateProducer, - syncProducer, eduProducer, transactionsCache, fedSenderAPI, + syncProducer, eduProducer, transactionsCache, fsAPI, ) } diff --git a/clientapi/consumers/roomserver.go b/clientapi/consumers/roomserver.go index 6d5bb09a6..d0c91e88f 100644 --- a/clientapi/consumers/roomserver.go +++ b/clientapi/consumers/roomserver.go @@ -24,16 +24,16 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/Shopify/sarama" log "github.com/sirupsen/logrus" - sarama "gopkg.in/Shopify/sarama.v1" ) // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { - roomServerConsumer *common.ContinualConsumer - db accounts.Database - query api.RoomserverQueryAPI - serverName string + rsAPI api.RoomserverInternalAPI + rsConsumer *common.ContinualConsumer + db accounts.Database + serverName string } // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. @@ -41,7 +41,7 @@ func NewOutputRoomEventConsumer( cfg *config.Dendrite, kafkaConsumer sarama.Consumer, store accounts.Database, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, ) *OutputRoomEventConsumer { consumer := common.ContinualConsumer{ @@ -50,10 +50,10 @@ func NewOutputRoomEventConsumer( PartitionStore: store, } s := &OutputRoomEventConsumer{ - roomServerConsumer: &consumer, - db: store, - query: queryAPI, - serverName: string(cfg.Matrix.ServerName), + rsConsumer: &consumer, + db: store, + rsAPI: rsAPI, + serverName: string(cfg.Matrix.ServerName), } consumer.ProcessMessage = s.onMessage @@ -62,7 +62,7 @@ func NewOutputRoomEventConsumer( // Start consuming from room servers func (s *OutputRoomEventConsumer) Start() error { - return s.roomServerConsumer.Start() + return s.rsConsumer.Start() } // onMessage is called when the sync server receives a new event from the room server output log. @@ -134,7 +134,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( // Request the missing events from the roomserver eventReq := api.QueryEventsByIDRequest{EventIDs: missing} var eventResp api.QueryEventsByIDResponse - if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { + if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { return nil, err } diff --git a/clientapi/jsonerror/jsonerror.go b/clientapi/jsonerror/jsonerror.go index 735de5bea..85e887aec 100644 --- a/clientapi/jsonerror/jsonerror.go +++ b/clientapi/jsonerror/jsonerror.go @@ -18,6 +18,7 @@ import ( "fmt" "net/http" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -124,6 +125,12 @@ func GuestAccessForbidden(msg string) *MatrixError { return &MatrixError{"M_GUEST_ACCESS_FORBIDDEN", msg} } +// IncompatibleRoomVersion is an error which is returned when the client +// requests a room with a version that is unsupported. +func IncompatibleRoomVersion(roomVersion gomatrixserverlib.RoomVersion) *MatrixError { + return &MatrixError{"M_INCOMPATIBLE_ROOM_VERSION", string(roomVersion)} +} + // UnsupportedRoomVersion is an error which is returned when the client // requests a room with a version that is unsupported. func UnsupportedRoomVersion(msg string) *MatrixError { diff --git a/clientapi/producers/roomserver.go b/clientapi/producers/roomserver.go index 391ea07bf..a804abfe9 100644 --- a/clientapi/producers/roomserver.go +++ b/clientapi/producers/roomserver.go @@ -23,15 +23,13 @@ import ( // RoomserverProducer produces events for the roomserver to consume. type RoomserverProducer struct { - InputAPI api.RoomserverInputAPI - QueryAPI api.RoomserverQueryAPI + RsAPI api.RoomserverInternalAPI } // NewRoomserverProducer creates a new RoomserverProducer -func NewRoomserverProducer(inputAPI api.RoomserverInputAPI, queryAPI api.RoomserverQueryAPI) *RoomserverProducer { +func NewRoomserverProducer(rsAPI api.RoomserverInternalAPI) *RoomserverProducer { return &RoomserverProducer{ - InputAPI: inputAPI, - QueryAPI: queryAPI, + RsAPI: rsAPI, } } @@ -95,7 +93,7 @@ func (c *RoomserverProducer) SendInputRoomEvents( ) (eventID string, err error) { request := api.InputRoomEventsRequest{InputRoomEvents: ires} var response api.InputRoomEventsResponse - err = c.InputAPI.InputRoomEvents(ctx, &request, &response) + err = c.RsAPI.InputRoomEvents(ctx, &request, &response) eventID = response.EventID return } @@ -106,14 +104,17 @@ func (c *RoomserverProducer) SendInputRoomEvents( func (c *RoomserverProducer) SendInvite( ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, inviteRoomState []gomatrixserverlib.InviteV2StrippedState, + sendAsServer gomatrixserverlib.ServerName, txnID *api.TransactionID, ) error { request := api.InputRoomEventsRequest{ InputInviteEvents: []api.InputInviteEvent{{ Event: inviteEvent, InviteRoomState: inviteRoomState, RoomVersion: inviteEvent.RoomVersion, + SendAsServer: string(sendAsServer), + TransactionID: txnID, }}, } var response api.InputRoomEventsResponse - return c.InputAPI.InputRoomEvents(ctx, &request, &response) + return c.RsAPI.InputRoomEvents(ctx, &request, &response) } diff --git a/clientapi/producers/syncapi.go b/clientapi/producers/syncapi.go index 6bfcd51aa..0a446e2f5 100644 --- a/clientapi/producers/syncapi.go +++ b/clientapi/producers/syncapi.go @@ -19,7 +19,7 @@ import ( "github.com/matrix-org/dendrite/common" - sarama "gopkg.in/Shopify/sarama.v1" + "github.com/Shopify/sarama" ) // SyncAPIProducer produces events for the sync API server to consume diff --git a/clientapi/producers/userupdate.go b/clientapi/producers/userupdate.go index 2a5dfc70a..426b6d509 100644 --- a/clientapi/producers/userupdate.go +++ b/clientapi/producers/userupdate.go @@ -17,7 +17,7 @@ package producers import ( "encoding/json" - sarama "gopkg.in/Shopify/sarama.v1" + "github.com/Shopify/sarama" ) // UserUpdateProducer produces events related to user updates. diff --git a/clientapi/routing/capabilities.go b/clientapi/routing/capabilities.go index 1792c6308..199b15240 100644 --- a/clientapi/routing/capabilities.go +++ b/clientapi/routing/capabilities.go @@ -26,11 +26,11 @@ import ( // SendMembership implements PUT /rooms/{roomID}/(join|kick|ban|unban|leave|invite) // by building a m.room.member event then sending it to the room server func GetCapabilities( - req *http.Request, queryAPI roomserverAPI.RoomserverQueryAPI, + req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, ) util.JSONResponse { roomVersionsQueryReq := roomserverAPI.QueryRoomVersionCapabilitiesRequest{} roomVersionsQueryRes := roomserverAPI.QueryRoomVersionCapabilitiesResponse{} - if err := queryAPI.QueryRoomVersionCapabilities( + if err := rsAPI.QueryRoomVersionCapabilities( req.Context(), &roomVersionsQueryReq, &roomVersionsQueryRes, diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 2953e4e54..424260e45 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -164,13 +164,13 @@ type fledglingEvent struct { func CreateRoom( req *http.Request, device *authtypes.Device, cfg *config.Dendrite, producer *producers.RoomserverProducer, - accountDB accounts.Database, aliasAPI roomserverAPI.RoomserverAliasAPI, + accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { // TODO (#267): Check room ID doesn't clash with an existing one, and we // probably shouldn't be using pseudo-random strings, maybe GUIDs? roomID := fmt.Sprintf("!%s:%s", util.RandomString(16), cfg.Matrix.ServerName) - return createRoom(req, device, cfg, roomID, producer, accountDB, aliasAPI, asAPI) + return createRoom(req, device, cfg, roomID, producer, accountDB, rsAPI, asAPI) } // createRoom implements /createRoom @@ -178,7 +178,7 @@ func CreateRoom( func createRoom( req *http.Request, device *authtypes.Device, cfg *config.Dendrite, roomID string, producer *producers.RoomserverProducer, - accountDB accounts.Database, aliasAPI roomserverAPI.RoomserverAliasAPI, + accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { logger := util.GetLogger(req.Context()) @@ -400,7 +400,7 @@ func createRoom( } var aliasResp roomserverAPI.SetRoomAliasResponse - err = aliasAPI.SetRoomAlias(req.Context(), &aliasReq, &aliasResp) + err = rsAPI.SetRoomAlias(req.Context(), &aliasReq, &aliasResp) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.SetRoomAlias failed") return jsonerror.InternalServerError() diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index 248696ab2..a0a60a47a 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -46,8 +46,8 @@ func DirectoryRoom( roomAlias string, federation *gomatrixserverlib.FederationClient, cfg *config.Dendrite, - rsAPI roomserverAPI.RoomserverAliasAPI, - fedSenderAPI federationSenderAPI.FederationSenderQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, + fedSenderAPI federationSenderAPI.FederationSenderInternalAPI, ) util.JSONResponse { _, domain, err := gomatrixserverlib.SplitID('#', roomAlias) if err != nil { @@ -115,7 +115,7 @@ func SetLocalAlias( device *authtypes.Device, alias string, cfg *config.Dendrite, - aliasAPI roomserverAPI.RoomserverAliasAPI, + aliasAPI roomserverAPI.RoomserverInternalAPI, ) util.JSONResponse { _, domain, err := gomatrixserverlib.SplitID('#', alias) if err != nil { @@ -190,7 +190,7 @@ func RemoveLocalAlias( req *http.Request, device *authtypes.Device, alias string, - aliasAPI roomserverAPI.RoomserverAliasAPI, + aliasAPI roomserverAPI.RoomserverInternalAPI, ) util.JSONResponse { creatorQueryReq := roomserverAPI.GetCreatorIDForAliasRequest{ diff --git a/clientapi/routing/getevent.go b/clientapi/routing/getevent.go index 2d3152510..bf49968de 100644 --- a/clientapi/routing/getevent.go +++ b/clientapi/routing/getevent.go @@ -44,7 +44,7 @@ func GetEvent( roomID string, eventID string, cfg *config.Dendrite, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, federation *gomatrixserverlib.FederationClient, keyRing gomatrixserverlib.KeyRing, ) util.JSONResponse { @@ -52,7 +52,7 @@ func GetEvent( EventIDs: []string{eventID}, } var eventsResp api.QueryEventsByIDResponse - err := queryAPI.QueryEventsByID(req.Context(), &eventsReq, &eventsResp) + err := rsAPI.QueryEventsByID(req.Context(), &eventsReq, &eventsResp) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryEventsByID failed") return jsonerror.InternalServerError() @@ -88,7 +88,7 @@ func GetEvent( }}, } var stateResp api.QueryStateAfterEventsResponse - if err := queryAPI.QueryStateAfterEvents(req.Context(), &stateReq, &stateResp); err != nil { + if err := rsAPI.QueryStateAfterEvents(req.Context(), &stateReq, &stateResp); err != nil { util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryStateAfterEvents failed") return jsonerror.InternalServerError() } diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index f72bb9162..48e42214d 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -15,434 +15,48 @@ package routing import ( - "fmt" "net/http" - "strings" - "time" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/clientapi/producers" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" - "github.com/matrix-org/dendrite/roomserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/gomatrix" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" - "github.com/sirupsen/logrus" ) -// JoinRoomByIDOrAlias implements the "/join/{roomIDOrAlias}" API. -// https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-join-roomidoralias func JoinRoomByIDOrAlias( req *http.Request, device *authtypes.Device, + rsAPI roomserverAPI.RoomserverInternalAPI, roomIDOrAlias string, - cfg *config.Dendrite, - federation *gomatrixserverlib.FederationClient, - producer *producers.RoomserverProducer, - queryAPI roomserverAPI.RoomserverQueryAPI, - aliasAPI roomserverAPI.RoomserverAliasAPI, - keyRing gomatrixserverlib.KeyRing, - accountDB accounts.Database, ) util.JSONResponse { - var content map[string]interface{} // must be a JSON object - if resErr := httputil.UnmarshalJSONRequest(req, &content); resErr != nil { - return *resErr + // Prepare to ask the roomserver to perform the room join. + joinReq := roomserverAPI.PerformJoinRequest{ + RoomIDOrAlias: roomIDOrAlias, + UserID: device.UserID, + } + joinRes := roomserverAPI.PerformJoinResponse{} + + // If content was provided in the request then incude that + // in the request. It'll get used as a part of the membership + // event content. + if err := httputil.UnmarshalJSONRequest(req, &joinReq.Content); err != nil { + return *err } - evTime, err := httputil.ParseTSParam(req) - if err != nil { + // Ask the roomserver to perform the join. + if err := rsAPI.PerformJoin(req.Context(), &joinReq, &joinRes); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: jsonerror.InvalidArgumentValue(err.Error()), + JSON: jsonerror.Unknown(err.Error()), } } - localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") - return jsonerror.InternalServerError() - } - - profile, err := accountDB.GetProfileByLocalpart(req.Context(), localpart) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetProfileByLocalpart failed") - return jsonerror.InternalServerError() - } - - content["membership"] = gomatrixserverlib.Join - content["displayname"] = profile.DisplayName - content["avatar_url"] = profile.AvatarURL - - r := joinRoomReq{ - req, evTime, content, device.UserID, cfg, federation, producer, queryAPI, aliasAPI, keyRing, - } - - if strings.HasPrefix(roomIDOrAlias, "!") { - return r.joinRoomByID(roomIDOrAlias) - } - if strings.HasPrefix(roomIDOrAlias, "#") { - return r.joinRoomByAlias(roomIDOrAlias) - } return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON( - fmt.Sprintf("Invalid first character '%s' for room ID or alias", - string([]rune(roomIDOrAlias)[0])), // Wrapping with []rune makes this call UTF-8 safe - ), - } -} - -type joinRoomReq struct { - req *http.Request - evTime time.Time - content map[string]interface{} - userID string - cfg *config.Dendrite - federation *gomatrixserverlib.FederationClient - producer *producers.RoomserverProducer - queryAPI roomserverAPI.RoomserverQueryAPI - aliasAPI roomserverAPI.RoomserverAliasAPI - keyRing gomatrixserverlib.KeyRing -} - -// joinRoomByID joins a room by room ID -func (r joinRoomReq) joinRoomByID(roomID string) util.JSONResponse { - // A client should only join a room by room ID when it has an invite - // to the room. If the server is already in the room then we can - // lookup the invite and process the request as a normal state event. - // If the server is not in the room the we will need to look up the - // remote server the invite came from in order to request a join event - // from that server. - queryReq := roomserverAPI.QueryInvitesForUserRequest{ - RoomID: roomID, TargetUserID: r.userID, - } - var queryRes roomserverAPI.QueryInvitesForUserResponse - if err := r.queryAPI.QueryInvitesForUser(r.req.Context(), &queryReq, &queryRes); err != nil { - util.GetLogger(r.req.Context()).WithError(err).Error("r.queryAPI.QueryInvitesForUser failed") - return jsonerror.InternalServerError() - } - - servers := []gomatrixserverlib.ServerName{} - seenInInviterIDs := map[gomatrixserverlib.ServerName]bool{} - for _, userID := range queryRes.InviteSenderUserIDs { - _, domain, err := gomatrixserverlib.SplitID('@', userID) - if err != nil { - util.GetLogger(r.req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") - return jsonerror.InternalServerError() - } - if !seenInInviterIDs[domain] { - servers = append(servers, domain) - seenInInviterIDs[domain] = true - } - } - - // Also add the domain extracted from the roomID as a last resort to join - // in case the client is erroneously trying to join by ID without an invite - // or all previous attempts at domains extracted from the inviter IDs fail - // Note: It's no guarantee we'll succeed because a room isn't bound to the domain in its ID - _, domain, err := gomatrixserverlib.SplitID('!', roomID) - if err != nil { - util.GetLogger(r.req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") - return jsonerror.InternalServerError() - } - if domain != r.cfg.Matrix.ServerName && !seenInInviterIDs[domain] { - servers = append(servers, domain) - } - - return r.joinRoomUsingServers(roomID, servers) - -} - -// joinRoomByAlias joins a room using a room alias. -func (r joinRoomReq) joinRoomByAlias(roomAlias string) util.JSONResponse { - _, domain, err := gomatrixserverlib.SplitID('#', roomAlias) - if err != nil { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON("Room alias must be in the form '#localpart:domain'"), - } - } - if domain == r.cfg.Matrix.ServerName { - queryReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: roomAlias} - var queryRes roomserverAPI.GetRoomIDForAliasResponse - if err = r.aliasAPI.GetRoomIDForAlias(r.req.Context(), &queryReq, &queryRes); err != nil { - util.GetLogger(r.req.Context()).WithError(err).Error("r.aliasAPI.GetRoomIDForAlias failed") - return jsonerror.InternalServerError() - } - - if len(queryRes.RoomID) > 0 { - return r.joinRoomUsingServers(queryRes.RoomID, []gomatrixserverlib.ServerName{r.cfg.Matrix.ServerName}) - } - // If the response doesn't contain a non-empty string, return an error - return util.JSONResponse{ - Code: http.StatusNotFound, - JSON: jsonerror.NotFound("Room alias " + roomAlias + " not found."), - } - } - // If the room isn't local, use federation to join - return r.joinRoomByRemoteAlias(domain, roomAlias) -} - -func (r joinRoomReq) joinRoomByRemoteAlias( - domain gomatrixserverlib.ServerName, roomAlias string, -) util.JSONResponse { - resp, err := r.federation.LookupRoomAlias(r.req.Context(), domain, roomAlias) - if err != nil { - switch x := err.(type) { - case gomatrix.HTTPError: - if x.Code == http.StatusNotFound { - return util.JSONResponse{ - Code: http.StatusNotFound, - JSON: jsonerror.NotFound("Room alias not found"), - } - } - } - util.GetLogger(r.req.Context()).WithError(err).Error("r.federation.LookupRoomAlias failed") - return jsonerror.InternalServerError() - } - - return r.joinRoomUsingServers(resp.RoomID, resp.Servers) -} - -func (r joinRoomReq) writeToBuilder(eb *gomatrixserverlib.EventBuilder, roomID string) error { - eb.Type = "m.room.member" - - err := eb.SetContent(r.content) - if err != nil { - return err - } - - err = eb.SetUnsigned(struct{}{}) - if err != nil { - return err - } - - eb.Sender = r.userID - eb.StateKey = &r.userID - eb.RoomID = roomID - eb.Redacts = "" - - return nil -} - -func (r joinRoomReq) joinRoomUsingServers( - roomID string, servers []gomatrixserverlib.ServerName, -) util.JSONResponse { - var eb gomatrixserverlib.EventBuilder - err := r.writeToBuilder(&eb, roomID) - if err != nil { - util.GetLogger(r.req.Context()).WithError(err).Error("r.writeToBuilder failed") - return jsonerror.InternalServerError() - } - - queryRes := roomserverAPI.QueryLatestEventsAndStateResponse{} - event, err := common.BuildEvent(r.req.Context(), &eb, r.cfg, r.evTime, r.queryAPI, &queryRes) - if err == nil { - // If we have successfully built an event at this point then we can - // assert that the room is a local room, as BuildEvent was able to - // add prev_events etc successfully. - if _, err = r.producer.SendEvents( - r.req.Context(), - []gomatrixserverlib.HeaderedEvent{ - (*event).Headered(queryRes.RoomVersion), - }, - r.cfg.Matrix.ServerName, - nil, - ); err != nil { - util.GetLogger(r.req.Context()).WithError(err).Error("r.producer.SendEvents failed") - return jsonerror.InternalServerError() - } - return util.JSONResponse{ - Code: http.StatusOK, - JSON: struct { - RoomID string `json:"room_id"` - }{roomID}, - } - } - - // Otherwise, if we've reached here, then we haven't been able to populate - // prev_events etc for the room, therefore the room is probably federated. - - // TODO: This needs to be re-thought, as in the case of an invite, the room - // will exist in the database in roomserver_rooms but won't have any state - // events, therefore this below check fails. - if err != common.ErrRoomNoExists { - util.GetLogger(r.req.Context()).WithError(err).Error("common.BuildEvent failed") - return jsonerror.InternalServerError() - } - - if len(servers) == 0 { - return util.JSONResponse{ - Code: http.StatusNotFound, - JSON: jsonerror.NotFound("No candidate servers found for room"), - } - } - - var lastErr error - for _, server := range servers { - var response *util.JSONResponse - response, lastErr = r.joinRoomUsingServer(roomID, server) - if lastErr != nil { - // There was a problem talking to one of the servers. - util.GetLogger(r.req.Context()).WithError(lastErr).WithField("server", server).Warn("Failed to join room using server") - // Try the next server. - if r.req.Context().Err() != nil { - // The request context has expired so don't bother trying any - // more servers - they will immediately fail due to the expired - // context. - break - } else { - // The request context hasn't expired yet so try the next server. - continue - } - } - return *response - } - - // Every server we tried to join through resulted in an error. - // We return the error from the last server. - - // TODO: Generate the correct HTTP status code for all different - // kinds of errors that could have happened. - // The possible errors include: - // 1) We can't connect to the remote servers. - // 2) None of the servers we could connect to think we are allowed - // to join the room. - // 3) The remote server returned something invalid. - // 4) We couldn't fetch the public keys needed to verify the - // signatures on the state events. - // 5) ... - util.GetLogger(r.req.Context()).WithError(lastErr).Error("failed to join through any server") - return jsonerror.InternalServerError() -} - -// joinRoomUsingServer tries to join a remote room using a given matrix server. -// If there was a failure communicating with the server or the response from the -// server was invalid this returns an error. -// Otherwise this returns a JSONResponse. -func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib.ServerName) (*util.JSONResponse, error) { - // Ask the room server for information about room versions. - var request api.QueryRoomVersionCapabilitiesRequest - var response api.QueryRoomVersionCapabilitiesResponse - if err := r.queryAPI.QueryRoomVersionCapabilities(r.req.Context(), &request, &response); err != nil { - return nil, err - } - var supportedVersions []gomatrixserverlib.RoomVersion - for version := range response.AvailableRoomVersions { - supportedVersions = append(supportedVersions, version) - } - respMakeJoin, err := r.federation.MakeJoin(r.req.Context(), server, roomID, r.userID, supportedVersions) - if err != nil { - // TODO: Check if the user was not allowed to join the room. - return nil, fmt.Errorf("r.federation.MakeJoin: %w", err) - } - - // Set all the fields to be what they should be, this should be a no-op - // but it's possible that the remote server returned us something "odd" - err = r.writeToBuilder(&respMakeJoin.JoinEvent, roomID) - if err != nil { - return nil, fmt.Errorf("r.writeToBuilder: %w", err) - } - - if respMakeJoin.RoomVersion == "" { - respMakeJoin.RoomVersion = gomatrixserverlib.RoomVersionV1 - } - if _, err = respMakeJoin.RoomVersion.EventFormat(); err != nil { - return &util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.UnsupportedRoomVersion( - fmt.Sprintf("Room version '%s' is not supported", respMakeJoin.RoomVersion), - ), - }, nil - } - - event, err := respMakeJoin.JoinEvent.Build( - r.evTime, r.cfg.Matrix.ServerName, r.cfg.Matrix.KeyID, - r.cfg.Matrix.PrivateKey, respMakeJoin.RoomVersion, - ) - if err != nil { - return nil, fmt.Errorf("respMakeJoin.JoinEvent.Build: %w", err) - } - - respSendJoin, err := r.federation.SendJoin(r.req.Context(), server, event, respMakeJoin.RoomVersion) - if err != nil { - return nil, fmt.Errorf("r.federation.SendJoin: %w", err) - } - - if err = r.checkSendJoinResponse(event, server, respMakeJoin, respSendJoin); err != nil { - return nil, err - } - - util.GetLogger(r.req.Context()).WithFields(logrus.Fields{ - "room_id": roomID, - "num_auth_events": len(respSendJoin.AuthEvents), - "num_state_events": len(respSendJoin.StateEvents), - }).Info("Room join signature and auth verification passed") - - if err = r.producer.SendEventWithState( - r.req.Context(), - gomatrixserverlib.RespState(respSendJoin.RespState), - event.Headered(respMakeJoin.RoomVersion), - ); err != nil { - util.GetLogger(r.req.Context()).WithError(err).Error("r.producer.SendEventWithState") - } - - return &util.JSONResponse{ Code: http.StatusOK, // TODO: Put the response struct somewhere common. JSON: struct { RoomID string `json:"room_id"` - }{roomID}, - }, nil -} - -// checkSendJoinResponse checks that all of the signatures are correct -// and that the join is allowed by the supplied state. -func (r joinRoomReq) checkSendJoinResponse( - event gomatrixserverlib.Event, - server gomatrixserverlib.ServerName, - respMakeJoin gomatrixserverlib.RespMakeJoin, - respSendJoin gomatrixserverlib.RespSendJoin, -) error { - // A list of events that we have retried, if they were not included in - // the auth events supplied in the send_join. - retries := map[string]bool{} - -retryCheck: - // TODO: Can we expand Check here to return a list of missing auth - // events rather than failing one at a time? - if err := respSendJoin.Check(r.req.Context(), r.keyRing, event); err != nil { - switch e := err.(type) { - case gomatrixserverlib.MissingAuthEventError: - // Check that we haven't already retried for this event, prevents - // us from ending up in endless loops - if !retries[e.AuthEventID] { - // Ask the server that we're talking to right now for the event - tx, txerr := r.federation.GetEvent(r.req.Context(), server, e.AuthEventID) - if txerr != nil { - return fmt.Errorf("r.federation.GetEvent: %w", txerr) - } - // For each event returned, add it to the auth events. - for _, pdu := range tx.PDUs { - ev, everr := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, respMakeJoin.RoomVersion) - if everr != nil { - return fmt.Errorf("gomatrixserverlib.NewEventFromUntrustedJSON: %w", everr) - } - respSendJoin.AuthEvents = append(respSendJoin.AuthEvents, ev) - } - // Mark the event as retried and then give the check another go. - retries[e.AuthEventID] = true - goto retryCheck - } - return fmt.Errorf("respSendJoin (after retries): %w", e) - default: - return fmt.Errorf("respSendJoin: %w", err) - } + }{joinReq.RoomIDOrAlias}, } - return nil } diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 9f386b718..9030f9f7e 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -40,15 +40,17 @@ var errMissingUserID = errors.New("'user_id' must be supplied") // SendMembership implements PUT /rooms/{roomID}/(join|kick|ban|unban|leave|invite) // by building a m.room.member event then sending it to the room server +// TODO: Can we improve the cyclo count here? Separate code paths for invites? +// nolint:gocyclo func SendMembership( req *http.Request, accountDB accounts.Database, device *authtypes.Device, roomID string, membership string, cfg *config.Dendrite, - queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, producer *producers.RoomserverProducer, ) util.JSONResponse { verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := queryAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { + if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UnsupportedRoomVersion(err.Error()), @@ -69,7 +71,7 @@ func SendMembership( } inviteStored, jsonErrResp := checkAndProcessThreepid( - req, device, &body, cfg, queryAPI, accountDB, producer, + req, device, &body, cfg, rsAPI, accountDB, producer, membership, roomID, evTime, ) if jsonErrResp != nil { @@ -87,7 +89,7 @@ func SendMembership( } event, err := buildMembershipEvent( - req.Context(), body, accountDB, device, membership, roomID, cfg, evTime, queryAPI, asAPI, + req.Context(), body, accountDB, device, membership, roomID, cfg, evTime, rsAPI, asAPI, ) if err == errMissingUserID { return util.JSONResponse{ @@ -104,23 +106,39 @@ func SendMembership( return jsonerror.InternalServerError() } - if _, err := producer.SendEvents( - req.Context(), - []gomatrixserverlib.HeaderedEvent{(*event).Headered(verRes.RoomVersion)}, - cfg.Matrix.ServerName, - nil, - ); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed") - return jsonerror.InternalServerError() - } - var returnData interface{} = struct{}{} - // The join membership requires the room id to be sent in the response - if membership == gomatrixserverlib.Join { + switch membership { + case gomatrixserverlib.Invite: + // Invites need to be handled specially + err = producer.SendInvite( + req.Context(), + event.Headered(verRes.RoomVersion), + nil, // ask the roomserver to draw up invite room state for us + cfg.Matrix.ServerName, + nil, + ) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("producer.SendInvite failed") + return jsonerror.InternalServerError() + } + case gomatrixserverlib.Join: + // The join membership requires the room id to be sent in the response returnData = struct { RoomID string `json:"room_id"` }{roomID} + fallthrough + default: + _, err = producer.SendEvents( + req.Context(), + []gomatrixserverlib.HeaderedEvent{event.Headered(verRes.RoomVersion)}, + cfg.Matrix.ServerName, + nil, + ) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed") + return jsonerror.InternalServerError() + } } return util.JSONResponse{ @@ -135,7 +153,7 @@ func buildMembershipEvent( device *authtypes.Device, membership, roomID string, cfg *config.Dendrite, evTime time.Time, - queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) (*gomatrixserverlib.Event, error) { stateKey, reason, err := getMembershipStateKey(body, device, membership) if err != nil { @@ -170,7 +188,7 @@ func buildMembershipEvent( return nil, err } - return common.BuildEvent(ctx, &builder, cfg, evTime, queryAPI, nil) + return common.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil) } // loadProfile lookups the profile of a given user from the database and returns @@ -230,7 +248,7 @@ func checkAndProcessThreepid( device *authtypes.Device, body *threepid.MembershipRequest, cfg *config.Dendrite, - queryAPI roomserverAPI.RoomserverQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, accountDB accounts.Database, producer *producers.RoomserverProducer, membership, roomID string, @@ -238,7 +256,7 @@ func checkAndProcessThreepid( ) (inviteStored bool, errRes *util.JSONResponse) { inviteStored, err := threepid.CheckAndProcessInvite( - req.Context(), device, body, cfg, queryAPI, accountDB, producer, + req.Context(), device, body, cfg, rsAPI, accountDB, producer, membership, roomID, evTime, ) if err == threepid.ErrMissingParameter { diff --git a/clientapi/routing/memberships.go b/clientapi/routing/memberships.go index 0b846e5e3..f5d9bc4c5 100644 --- a/clientapi/routing/memberships.go +++ b/clientapi/routing/memberships.go @@ -39,7 +39,7 @@ type getJoinedRoomsResponse struct { func GetMemberships( req *http.Request, device *authtypes.Device, roomID string, joinedOnly bool, _ *config.Dendrite, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { queryReq := api.QueryMembershipsForRoomRequest{ JoinedOnly: joinedOnly, @@ -47,8 +47,8 @@ func GetMemberships( Sender: device.UserID, } var queryRes api.QueryMembershipsForRoomResponse - if err := queryAPI.QueryMembershipsForRoom(req.Context(), &queryReq, &queryRes); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryMembershipsForRoom failed") + if err := rsAPI.QueryMembershipsForRoom(req.Context(), &queryReq, &queryRes); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryMembershipsForRoom failed") return jsonerror.InternalServerError() } diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index a51c55ea5..b51533e45 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -94,7 +94,7 @@ func GetAvatarURL( func SetAvatarURL( req *http.Request, accountDB accounts.Database, device *authtypes.Device, userID string, producer *producers.UserUpdateProducer, cfg *config.Dendrite, - rsProducer *producers.RoomserverProducer, queryAPI api.RoomserverQueryAPI, + rsProducer *producers.RoomserverProducer, rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { if userID != device.UserID { return util.JSONResponse{ @@ -154,7 +154,7 @@ func SetAvatarURL( } events, err := buildMembershipEvents( - req.Context(), memberships, newProfile, userID, cfg, evTime, queryAPI, + req.Context(), memberships, newProfile, userID, cfg, evTime, rsAPI, ) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvents failed") @@ -208,7 +208,7 @@ func GetDisplayName( func SetDisplayName( req *http.Request, accountDB accounts.Database, device *authtypes.Device, userID string, producer *producers.UserUpdateProducer, cfg *config.Dendrite, - rsProducer *producers.RoomserverProducer, queryAPI api.RoomserverQueryAPI, + rsProducer *producers.RoomserverProducer, rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { if userID != device.UserID { return util.JSONResponse{ @@ -268,7 +268,7 @@ func SetDisplayName( } events, err := buildMembershipEvents( - req.Context(), memberships, newProfile, userID, cfg, evTime, queryAPI, + req.Context(), memberships, newProfile, userID, cfg, evTime, rsAPI, ) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvents failed") @@ -337,14 +337,14 @@ func buildMembershipEvents( ctx context.Context, memberships []authtypes.Membership, newProfile authtypes.Profile, userID string, cfg *config.Dendrite, - evTime time.Time, queryAPI api.RoomserverQueryAPI, + evTime time.Time, rsAPI api.RoomserverInternalAPI, ) ([]gomatrixserverlib.HeaderedEvent, error) { evs := []gomatrixserverlib.HeaderedEvent{} for _, membership := range memberships { verReq := api.QueryRoomVersionForRoomRequest{RoomID: membership.RoomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := queryAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil { + if err := rsAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil { return []gomatrixserverlib.HeaderedEvent{}, err } @@ -366,7 +366,7 @@ func buildMembershipEvents( return nil, err } - event, err := common.BuildEvent(ctx, &builder, cfg, evTime, queryAPI, nil) + event, err := common.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil) if err != nil { return nil, err } diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 5dc6d7db9..3ceefa078 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -49,8 +49,7 @@ const pathPrefixUnstable = "/_matrix/client/unstable" func Setup( apiMux *mux.Router, cfg *config.Dendrite, producer *producers.RoomserverProducer, - queryAPI roomserverAPI.RoomserverQueryAPI, - aliasAPI roomserverAPI.RoomserverAliasAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, accountDB accounts.Database, deviceDB devices.Database, @@ -60,7 +59,7 @@ func Setup( syncProducer *producers.SyncAPIProducer, eduProducer *producers.EDUServerProducer, transactionsCache *transactions.Cache, - federationSender federationSenderAPI.FederationSenderQueryAPI, + federationSender federationSenderAPI.FederationSenderInternalAPI, ) { apiMux.Handle("/_matrix/client/versions", @@ -91,7 +90,7 @@ func Setup( r0mux.Handle("/createRoom", common.MakeAuthAPI("createRoom", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - return CreateRoom(req, device, cfg, producer, accountDB, aliasAPI, asAPI) + return CreateRoom(req, device, cfg, producer, accountDB, rsAPI, asAPI) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/join/{roomIDOrAlias}", @@ -101,7 +100,7 @@ func Setup( return util.ErrorResponse(err) } return JoinRoomByIDOrAlias( - req, device, vars["roomIDOrAlias"], cfg, federation, producer, queryAPI, aliasAPI, keyRing, accountDB, + req, device, rsAPI, vars["roomIDOrAlias"], ) }), ).Methods(http.MethodPost, http.MethodOptions) @@ -117,7 +116,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, queryAPI, asAPI, producer) + return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, rsAPI, asAPI, producer) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/send/{eventType}", @@ -126,7 +125,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, queryAPI, producer, nil) + return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, rsAPI, producer, nil) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}", @@ -137,7 +136,7 @@ func Setup( } txnID := vars["txnID"] return SendEvent(req, device, vars["roomID"], vars["eventType"], &txnID, - nil, cfg, queryAPI, producer, transactionsCache) + nil, cfg, rsAPI, producer, transactionsCache) }), ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/event/{eventID}", @@ -146,7 +145,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, queryAPI, federation, keyRing) + return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI, federation, keyRing) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -155,7 +154,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return OnIncomingStateRequest(req.Context(), queryAPI, vars["roomID"]) + return OnIncomingStateRequest(req.Context(), rsAPI, vars["roomID"]) })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{type}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { @@ -163,7 +162,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return OnIncomingStateTypeRequest(req.Context(), queryAPI, vars["roomID"], vars["type"], "") + return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], "") })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { @@ -171,7 +170,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return OnIncomingStateTypeRequest(req.Context(), queryAPI, vars["roomID"], vars["type"], vars["stateKey"]) + return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], vars["stateKey"]) })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}", @@ -186,7 +185,7 @@ func Setup( if strings.HasSuffix(eventType, "/") { eventType = eventType[:len(eventType)-1] } - return SendEvent(req, device, vars["roomID"], eventType, nil, &emptyString, cfg, queryAPI, producer, nil) + return SendEvent(req, device, vars["roomID"], eventType, nil, &emptyString, cfg, rsAPI, producer, nil) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -197,7 +196,7 @@ func Setup( return util.ErrorResponse(err) } stateKey := vars["stateKey"] - return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, &stateKey, cfg, queryAPI, producer, nil) + return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, &stateKey, cfg, rsAPI, producer, nil) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -219,7 +218,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return DirectoryRoom(req, vars["roomAlias"], federation, cfg, aliasAPI, federationSender) + return DirectoryRoom(req, vars["roomAlias"], federation, cfg, rsAPI, federationSender) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -229,7 +228,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SetLocalAlias(req, device, vars["roomAlias"], cfg, aliasAPI) + return SetLocalAlias(req, device, vars["roomAlias"], cfg, rsAPI) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -239,7 +238,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return RemoveLocalAlias(req, device, vars["roomAlias"], aliasAPI) + return RemoveLocalAlias(req, device, vars["roomAlias"], rsAPI) }), ).Methods(http.MethodDelete, http.MethodOptions) @@ -353,7 +352,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SetAvatarURL(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, queryAPI) + return SetAvatarURL(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, rsAPI) }), ).Methods(http.MethodPut, http.MethodOptions) // Browsers use the OPTIONS HTTP method to check if the CORS policy allows @@ -375,7 +374,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SetDisplayName(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, queryAPI) + return SetDisplayName(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, rsAPI) }), ).Methods(http.MethodPut, http.MethodOptions) // Browsers use the OPTIONS HTTP method to check if the CORS policy allows @@ -488,7 +487,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetMemberships(req, device, vars["roomID"], false, cfg, queryAPI) + return GetMemberships(req, device, vars["roomID"], false, cfg, rsAPI) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -498,7 +497,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetMemberships(req, device, vars["roomID"], true, cfg, queryAPI) + return GetMemberships(req, device, vars["roomID"], true, cfg, rsAPI) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -602,7 +601,7 @@ func Setup( r0mux.Handle("/capabilities", common.MakeAuthAPI("capabilities", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - return GetCapabilities(req, queryAPI) + return GetCapabilities(req, rsAPI) }), ).Methods(http.MethodGet) } diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index 5b2cd8ad4..7280dcd94 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -45,13 +45,13 @@ func SendEvent( device *authtypes.Device, roomID, eventType string, txnID, stateKey *string, cfg *config.Dendrite, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, producer *producers.RoomserverProducer, txnCache *transactions.Cache, ) util.JSONResponse { verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := queryAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { + if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UnsupportedRoomVersion(err.Error()), @@ -65,7 +65,7 @@ func SendEvent( } } - e, resErr := generateSendEvent(req, device, roomID, eventType, stateKey, cfg, queryAPI) + e, resErr := generateSendEvent(req, device, roomID, eventType, stateKey, cfg, rsAPI) if resErr != nil { return *resErr } @@ -115,7 +115,7 @@ func generateSendEvent( device *authtypes.Device, roomID, eventType string, stateKey *string, cfg *config.Dendrite, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, ) (*gomatrixserverlib.Event, *util.JSONResponse) { // parse the incoming http request userID := device.UserID @@ -148,7 +148,7 @@ func generateSendEvent( } var queryRes api.QueryLatestEventsAndStateResponse - e, err := common.BuildEvent(req.Context(), &builder, cfg, evTime, queryAPI, &queryRes) + e, err := common.BuildEvent(req.Context(), &builder, cfg, evTime, rsAPI, &queryRes) if err == common.ErrRoomNoExists { return nil, &util.JSONResponse{ Code: http.StatusNotFound, diff --git a/clientapi/routing/state.go b/clientapi/routing/state.go index c243eec0f..e3e5bdb6d 100644 --- a/clientapi/routing/state.go +++ b/clientapi/routing/state.go @@ -40,7 +40,7 @@ type stateEventInStateResp struct { // TODO: Check if the user is in the room. If not, check if the room's history // is publicly visible. Current behaviour is returning an empty array if the // user cannot see the room's history. -func OnIncomingStateRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI, roomID string) util.JSONResponse { +func OnIncomingStateRequest(ctx context.Context, rsAPI api.RoomserverInternalAPI, roomID string) util.JSONResponse { // TODO(#287): Auth request and handle the case where the user has left (where // we should return the state at the poin they left) stateReq := api.QueryLatestEventsAndStateRequest{ @@ -48,7 +48,7 @@ func OnIncomingStateRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI } stateRes := api.QueryLatestEventsAndStateResponse{} - if err := queryAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil { + if err := rsAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil { util.GetLogger(ctx).WithError(err).Error("queryAPI.QueryLatestEventsAndState failed") return jsonerror.InternalServerError() } @@ -98,7 +98,7 @@ func OnIncomingStateRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI // /rooms/{roomID}/state/{type}/{statekey} request. It will look in current // state to see if there is an event with that type and state key, if there // is then (by default) we return the content, otherwise a 404. -func OnIncomingStateTypeRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI, roomID string, evType, stateKey string) util.JSONResponse { +func OnIncomingStateTypeRequest(ctx context.Context, rsAPI api.RoomserverInternalAPI, roomID string, evType, stateKey string) util.JSONResponse { // TODO(#287): Auth request and handle the case where the user has left (where // we should return the state at the poin they left) util.GetLogger(ctx).WithFields(log.Fields{ @@ -118,7 +118,7 @@ func OnIncomingStateTypeRequest(ctx context.Context, queryAPI api.RoomserverQuer } stateRes := api.QueryLatestEventsAndStateResponse{} - if err := queryAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil { + if err := rsAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil { util.GetLogger(ctx).WithError(err).Error("queryAPI.QueryLatestEventsAndState failed") return jsonerror.InternalServerError() } diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index e34e91b56..5e7e4f2b4 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -87,7 +87,7 @@ var ( func CheckAndProcessInvite( ctx context.Context, device *authtypes.Device, body *MembershipRequest, cfg *config.Dendrite, - queryAPI api.RoomserverQueryAPI, db accounts.Database, + rsAPI api.RoomserverInternalAPI, db accounts.Database, producer *producers.RoomserverProducer, membership string, roomID string, evTime time.Time, ) (inviteStoredOnIDServer bool, err error) { @@ -112,7 +112,7 @@ func CheckAndProcessInvite( // "m.room.third_party_invite" have to be emitted from the data in // storeInviteRes. err = emit3PIDInviteEvent( - ctx, body, storeInviteRes, device, roomID, cfg, queryAPI, producer, evTime, + ctx, body, storeInviteRes, device, roomID, cfg, rsAPI, producer, evTime, ) inviteStoredOnIDServer = err == nil @@ -331,7 +331,7 @@ func emit3PIDInviteEvent( ctx context.Context, body *MembershipRequest, res *idServerStoreInviteResponse, device *authtypes.Device, roomID string, cfg *config.Dendrite, - queryAPI api.RoomserverQueryAPI, producer *producers.RoomserverProducer, + rsAPI api.RoomserverInternalAPI, producer *producers.RoomserverProducer, evTime time.Time, ) error { builder := &gomatrixserverlib.EventBuilder{ @@ -354,7 +354,7 @@ func emit3PIDInviteEvent( } queryRes := api.QueryLatestEventsAndStateResponse{} - event, err := common.BuildEvent(ctx, builder, cfg, evTime, queryAPI, &queryRes) + event, err := common.BuildEvent(ctx, builder, cfg, evTime, rsAPI, &queryRes) if err != nil { return err } diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go index fc51a5bb6..eca9b2fe6 100644 --- a/cmd/create-account/main.go +++ b/cmd/create-account/main.go @@ -63,7 +63,7 @@ func main() { serverName := gomatrixserverlib.ServerName(*serverNameStr) - accountDB, err := accounts.NewDatabase(*database, serverName) + accountDB, err := accounts.NewDatabase(*database, nil, serverName) if err != nil { fmt.Println(err.Error()) os.Exit(1) @@ -78,7 +78,7 @@ func main() { os.Exit(1) } - deviceDB, err := devices.NewDatabase(*database, serverName) + deviceDB, err := devices.NewDatabase(*database, nil, serverName) if err != nil { fmt.Println(err.Error()) os.Exit(1) diff --git a/cmd/dendrite-appservice-server/main.go b/cmd/dendrite-appservice-server/main.go index f203969f4..91ca707e7 100644 --- a/cmd/dendrite-appservice-server/main.go +++ b/cmd/dendrite-appservice-server/main.go @@ -28,11 +28,11 @@ func main() { accountDB := base.CreateAccountsDB() deviceDB := base.CreateDeviceDB() federation := base.CreateFederationClient() - alias, _, query := base.CreateHTTPRoomserverAPIs() + rsAPI := base.CreateHTTPRoomserverAPIs() cache := transactions.New() appservice.SetupAppServiceAPIComponent( - base, accountDB, deviceDB, federation, alias, query, cache, + base, accountDB, deviceDB, federation, rsAPI, cache, ) base.SetupAndServeHTTP(string(base.Cfg.Bind.AppServiceAPI), string(base.Cfg.Listen.AppServiceAPI)) diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index 815a978a8..1ce66db2c 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -36,13 +36,14 @@ func main() { keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) asQuery := base.CreateHTTPAppServiceAPIs() - alias, input, query := base.CreateHTTPRoomserverAPIs() - fedSenderAPI := base.CreateHTTPFederationSenderAPIs() + rsAPI := base.CreateHTTPRoomserverAPIs() + fsAPI := base.CreateHTTPFederationSenderAPIs() + rsAPI.SetFederationSenderAPI(fsAPI) eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, federation, &keyRing, - alias, input, query, eduInputAPI, asQuery, transactions.New(), fedSenderAPI, + rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, ) base.SetupAndServeHTTP(string(base.Cfg.Bind.ClientAPI), string(base.Cfg.Listen.ClientAPI)) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index f280c7483..0ff610a5b 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -56,6 +56,7 @@ func createKeyDB( ) keydb.Database { db, err := keydb.NewDatabase( string(base.Base.Cfg.Database.ServerKey), + base.Base.Cfg.DbProperties(), base.Base.Cfg.Matrix.ServerName, base.Base.Cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey), base.Base.Cfg.Matrix.KeyID, @@ -148,27 +149,34 @@ func main() { federation := createFederationClient(base) keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - alias, input, query := roomserver.SetupRoomServerComponent(&base.Base) - eduInputAPI := eduserver.SetupEDUServerComponent(&base.Base, cache.New()) - asQuery := appservice.SetupAppServiceAPIComponent( - &base.Base, accountDB, deviceDB, federation, alias, query, transactions.New(), + rsAPI := roomserver.SetupRoomServerComponent( + &base.Base, keyRing, federation, ) - fedSenderAPI := federationsender.SetupFederationSenderComponent(&base.Base, federation, query) + eduInputAPI := eduserver.SetupEDUServerComponent( + &base.Base, cache.New(), + ) + asAPI := appservice.SetupAppServiceAPIComponent( + &base.Base, accountDB, deviceDB, federation, rsAPI, transactions.New(), + ) + fsAPI := federationsender.SetupFederationSenderComponent( + &base.Base, federation, rsAPI, &keyRing, + ) + rsAPI.SetFederationSenderAPI(fsAPI) clientapi.SetupClientAPIComponent( &base.Base, deviceDB, accountDB, - federation, &keyRing, alias, input, query, - eduInputAPI, asQuery, transactions.New(), fedSenderAPI, + federation, &keyRing, rsAPI, + eduInputAPI, asAPI, transactions.New(), fsAPI, ) eduProducer := producers.NewEDUServerProducer(eduInputAPI) - federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer) + federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, &keyRing, rsAPI, asAPI, fsAPI, eduProducer) mediaapi.SetupMediaAPIComponent(&base.Base, deviceDB) publicRoomsDB, err := storage.NewPublicRoomsServerDatabaseWithPubSub(string(base.Base.Cfg.Database.PublicRoomsAPI), base.LibP2PPubsub) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - publicroomsapi.SetupPublicRoomsAPIComponent(&base.Base, deviceDB, publicRoomsDB, query, federation, nil) // Check this later - syncapi.SetupSyncAPIComponent(&base.Base, deviceDB, accountDB, query, federation, &cfg) + publicroomsapi.SetupPublicRoomsAPIComponent(&base.Base, deviceDB, publicRoomsDB, rsAPI, federation, nil) // Check this later + syncapi.SetupSyncAPIComponent(&base.Base, deviceDB, accountDB, rsAPI, federation, &cfg) httpHandler := common.WrapHandlerInCORS(base.Base.APIMux) diff --git a/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go b/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go index 819469ee8..cd2804c97 100644 --- a/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go +++ b/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go @@ -45,7 +45,7 @@ type PublicRoomsServerDatabase struct { // NewPublicRoomsServerDatabase creates a new public rooms server database. func NewPublicRoomsServerDatabase(dataSourceName string, dht *dht.IpfsDHT) (*PublicRoomsServerDatabase, error) { - pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName) + pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil) if err != nil { return nil, err } diff --git a/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go b/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go index 661192243..e4372c64f 100644 --- a/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go +++ b/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go @@ -48,7 +48,7 @@ type PublicRoomsServerDatabase struct { // NewPublicRoomsServerDatabase creates a new public rooms server database. func NewPublicRoomsServerDatabase(dataSourceName string, pubsub *pubsub.PubSub) (*PublicRoomsServerDatabase, error) { - pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName) + pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil) if err != nil { return nil, err } diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index dd06cd3f9..d829326a7 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -32,17 +32,18 @@ func main() { deviceDB := base.CreateDeviceDB() keyDB := base.CreateKeyDB() federation := base.CreateFederationClient() - federationSender := base.CreateHTTPFederationSenderAPIs() + fsAPI := base.CreateHTTPFederationSenderAPIs() keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - alias, input, query := base.CreateHTTPRoomserverAPIs() - asQuery := base.CreateHTTPAppServiceAPIs() + rsAPI := base.CreateHTTPRoomserverAPIs() + asAPI := base.CreateHTTPAppServiceAPIs() + rsAPI.SetFederationSenderAPI(fsAPI) eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) eduProducer := producers.NewEDUServerProducer(eduInputAPI) federationapi.SetupFederationAPIComponent( base, accountDB, deviceDB, federation, &keyRing, - alias, input, query, asQuery, federationSender, eduProducer, + rsAPI, asAPI, fsAPI, eduProducer, ) base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI)) diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-federation-sender-server/main.go index 71fc0b015..0daac1bcb 100644 --- a/cmd/dendrite-federation-sender-server/main.go +++ b/cmd/dendrite-federation-sender-server/main.go @@ -16,6 +16,7 @@ package main import ( "github.com/matrix-org/dendrite/common/basecomponent" + "github.com/matrix-org/dendrite/common/keydb" "github.com/matrix-org/dendrite/federationsender" ) @@ -25,12 +26,13 @@ func main() { defer base.Close() // nolint: errcheck federation := base.CreateFederationClient() - - _, _, query := base.CreateHTTPRoomserverAPIs() - - federationsender.SetupFederationSenderComponent( - base, federation, query, + keyDB := base.CreateKeyDB() + keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) + rsAPI := base.CreateHTTPRoomserverAPIs() + fsAPI := federationsender.SetupFederationSenderComponent( + base, federation, rsAPI, &keyRing, ) + rsAPI.SetFederationSenderAPI(fsAPI) base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationSender), string(base.Cfg.Listen.FederationSender)) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 6b0d83ae1..e004bc12e 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -57,27 +57,34 @@ func main() { federation := base.CreateFederationClient() keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - alias, input, query := roomserver.SetupRoomServerComponent(base) - eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) - asQuery := appservice.SetupAppServiceAPIComponent( - base, accountDB, deviceDB, federation, alias, query, transactions.New(), + rsAPI := roomserver.SetupRoomServerComponent( + base, keyRing, federation, ) - fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query) + eduInputAPI := eduserver.SetupEDUServerComponent( + base, cache.New(), + ) + asAPI := appservice.SetupAppServiceAPIComponent( + base, accountDB, deviceDB, federation, rsAPI, transactions.New(), + ) + fsAPI := federationsender.SetupFederationSenderComponent( + base, federation, rsAPI, &keyRing, + ) + rsAPI.SetFederationSenderAPI(fsAPI) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, - federation, &keyRing, alias, input, query, - eduInputAPI, asQuery, transactions.New(), fedSenderAPI, + federation, &keyRing, rsAPI, + eduInputAPI, asAPI, transactions.New(), fsAPI, ) eduProducer := producers.NewEDUServerProducer(eduInputAPI) - federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer) + federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asAPI, fsAPI, eduProducer) mediaapi.SetupMediaAPIComponent(base, deviceDB) - publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI)) + publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties()) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, query, federation, nil) - syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg) + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, federation, nil) + syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg) httpHandler := common.WrapHandlerInCORS(base.APIMux) diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go index f6a782f66..c3b49f4f7 100644 --- a/cmd/dendrite-public-rooms-api-server/main.go +++ b/cmd/dendrite-public-rooms-api-server/main.go @@ -28,12 +28,15 @@ func main() { deviceDB := base.CreateDeviceDB() - _, _, query := base.CreateHTTPRoomserverAPIs() - publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI)) + fsAPI := base.CreateHTTPFederationSenderAPIs() + rsAPI := base.CreateHTTPRoomserverAPIs() + rsAPI.SetFederationSenderAPI(fsAPI) + + publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties()) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, query, nil, nil) + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, nil, nil) base.SetupAndServeHTTP(string(base.Cfg.Bind.PublicRoomsAPI), string(base.Cfg.Listen.PublicRoomsAPI)) diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go index 41b705755..41149ad90 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-room-server/main.go @@ -18,6 +18,7 @@ import ( _ "net/http/pprof" "github.com/matrix-org/dendrite/common/basecomponent" + "github.com/matrix-org/dendrite/common/keydb" "github.com/matrix-org/dendrite/roomserver" ) @@ -25,8 +26,13 @@ func main() { cfg := basecomponent.ParseFlags() base := basecomponent.NewBaseDendrite(cfg, "RoomServerAPI") defer base.Close() // nolint: errcheck + keyDB := base.CreateKeyDB() + federation := base.CreateFederationClient() + keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - roomserver.SetupRoomServerComponent(base) + fsAPI := base.CreateHTTPFederationSenderAPIs() + rsAPI := roomserver.SetupRoomServerComponent(base, keyRing, federation) + rsAPI.SetFederationSenderAPI(fsAPI) base.SetupAndServeHTTP(string(base.Cfg.Bind.RoomServer), string(base.Cfg.Listen.RoomServer)) diff --git a/cmd/dendrite-sync-api-server/main.go b/cmd/dendrite-sync-api-server/main.go index 55e9faeef..259447af2 100644 --- a/cmd/dendrite-sync-api-server/main.go +++ b/cmd/dendrite-sync-api-server/main.go @@ -28,9 +28,9 @@ func main() { accountDB := base.CreateAccountsDB() federation := base.CreateFederationClient() - _, _, query := base.CreateHTTPRoomserverAPIs() + rsAPI := base.CreateHTTPRoomserverAPIs() - syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg) + syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg) base.SetupAndServeHTTP(string(base.Cfg.Bind.SyncAPI), string(base.Cfg.Listen.SyncAPI)) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 9bd8f2ee2..5b7ed4807 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -123,27 +123,28 @@ func main() { } p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node) - alias, input, query := roomserver.SetupRoomServerComponent(base) + rsAPI := roomserver.SetupRoomServerComponent(base, keyRing, federation) eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) asQuery := appservice.SetupAppServiceAPIComponent( - base, accountDB, deviceDB, federation, alias, query, transactions.New(), + base, accountDB, deviceDB, federation, rsAPI, transactions.New(), ) - fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query) + fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, rsAPI, &keyRing) + rsAPI.SetFederationSenderAPI(fedSenderAPI) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, - federation, &keyRing, alias, input, query, + federation, &keyRing, rsAPI, eduInputAPI, asQuery, transactions.New(), fedSenderAPI, ) eduProducer := producers.NewEDUServerProducer(eduInputAPI) - federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer) + federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asQuery, fedSenderAPI, eduProducer) mediaapi.SetupMediaAPIComponent(base, deviceDB) publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI)) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, query, federation, p2pPublicRoomProvider) - syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg) + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, federation, p2pPublicRoomProvider) + syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg) httpHandler := common.WrapHandlerInCORS(base.APIMux) diff --git a/cmd/kafka-producer/main.go b/cmd/kafka-producer/main.go index f5f243e4e..18ee3cdf2 100644 --- a/cmd/kafka-producer/main.go +++ b/cmd/kafka-producer/main.go @@ -21,7 +21,7 @@ import ( "os" "strings" - sarama "gopkg.in/Shopify/sarama.v1" + sarama "github.com/Shopify/sarama" ) const usage = `Usage: %s diff --git a/cmd/roomserver-integration-tests/main.go b/cmd/roomserver-integration-tests/main.go index 682fc6224..7126405e5 100644 --- a/cmd/roomserver-integration-tests/main.go +++ b/cmd/roomserver-integration-tests/main.go @@ -209,7 +209,7 @@ func writeToRoomServer(input []string, roomserverURL string) error { return err } } - x, err := api.NewRoomserverInputAPIHTTP(roomserverURL, &http.Client{Timeout: timeoutHTTP}) + x, err := api.NewRoomserverInternalAPIHTTP(roomserverURL, &http.Client{Timeout: timeoutHTTP}, nil) if err != nil { return err } @@ -225,7 +225,7 @@ func writeToRoomServer(input []string, roomserverURL string) error { // Once those messages have been written it runs the checkQueries function passing // a api.RoomserverQueryAPI client. The caller can use this function to check the // behaviour of the query API. -func testRoomserver(input []string, wantOutput []string, checkQueries func(api.RoomserverQueryAPI)) { +func testRoomserver(input []string, wantOutput []string, checkQueries func(api.RoomserverInternalAPI)) { dir, err := ioutil.TempDir("", "room-server-test") if err != nil { panic(err) @@ -276,7 +276,7 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R cmd.Args = []string{"dendrite-room-server", "--config", filepath.Join(dir, test.ConfigFile)} gotOutput, err := runAndReadFromTopic(cmd, cfg.RoomServerURL()+"/metrics", doInput, outputTopic, len(wantOutput), func() { - queryAPI, _ := api.NewRoomserverQueryAPIHTTP("http://"+string(cfg.Listen.RoomServer), &http.Client{Timeout: timeoutHTTP}, cache) + queryAPI, _ := api.NewRoomserverInternalAPIHTTP("http://"+string(cfg.Listen.RoomServer), &http.Client{Timeout: timeoutHTTP}, cache) checkQueries(queryAPI) }) if err != nil { @@ -410,7 +410,7 @@ func main() { }}`, } - testRoomserver(input, want, func(q api.RoomserverQueryAPI) { + testRoomserver(input, want, func(q api.RoomserverInternalAPI) { var response api.QueryLatestEventsAndStateResponse if err := q.QueryLatestEventsAndState( context.Background(), diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index 68a77cf99..a7e6736a6 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -33,8 +33,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/common" + "github.com/Shopify/sarama" "github.com/gorilla/mux" - sarama "gopkg.in/Shopify/sarama.v1" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/common/config" @@ -119,24 +119,12 @@ func (b *BaseDendrite) CreateHTTPAppServiceAPIs() appserviceAPI.AppServiceQueryA // CreateHTTPRoomserverAPIs returns the AliasAPI, InputAPI and QueryAPI for hitting // the roomserver over HTTP. -func (b *BaseDendrite) CreateHTTPRoomserverAPIs() ( - roomserverAPI.RoomserverAliasAPI, - roomserverAPI.RoomserverInputAPI, - roomserverAPI.RoomserverQueryAPI, -) { - alias, err := roomserverAPI.NewRoomserverAliasAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient) +func (b *BaseDendrite) CreateHTTPRoomserverAPIs() roomserverAPI.RoomserverInternalAPI { + rsAPI, err := roomserverAPI.NewRoomserverInternalAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient, b.ImmutableCache) if err != nil { - logrus.WithError(err).Panic("NewRoomserverAliasAPIHTTP failed") + logrus.WithError(err).Panic("NewRoomserverInternalAPIHTTP failed", b.httpClient) } - input, err := roomserverAPI.NewRoomserverInputAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient) - if err != nil { - logrus.WithError(err).Panic("NewRoomserverInputAPIHTTP failed", b.httpClient) - } - query, err := roomserverAPI.NewRoomserverQueryAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient, b.ImmutableCache) - if err != nil { - logrus.WithError(err).Panic("NewRoomserverQueryAPIHTTP failed", b.httpClient) - } - return alias, input, query + return rsAPI } // CreateHTTPEDUServerAPIs returns eduInputAPI for hitting the EDU @@ -149,12 +137,12 @@ func (b *BaseDendrite) CreateHTTPEDUServerAPIs() eduServerAPI.EDUServerInputAPI return e } -// CreateHTTPFederationSenderAPIs returns FederationSenderQueryAPI for hitting +// CreateHTTPFederationSenderAPIs returns FederationSenderInternalAPI for hitting // the federation sender over HTTP -func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.FederationSenderQueryAPI { - f, err := federationSenderAPI.NewFederationSenderQueryAPIHTTP(b.Cfg.FederationSenderURL(), b.httpClient) +func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.FederationSenderInternalAPI { + f, err := federationSenderAPI.NewFederationSenderInternalAPIHTTP(b.Cfg.FederationSenderURL(), b.httpClient) if err != nil { - logrus.WithError(err).Panic("NewFederationSenderQueryAPIHTTP failed", b.httpClient) + logrus.WithError(err).Panic("NewFederationSenderInternalAPIHTTP failed", b.httpClient) } return f } @@ -162,7 +150,7 @@ func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.Fede // CreateDeviceDB creates a new instance of the device database. Should only be // called once per component. func (b *BaseDendrite) CreateDeviceDB() devices.Database { - db, err := devices.NewDatabase(string(b.Cfg.Database.Device), b.Cfg.Matrix.ServerName) + db, err := devices.NewDatabase(string(b.Cfg.Database.Device), b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to devices db") } @@ -173,7 +161,7 @@ func (b *BaseDendrite) CreateDeviceDB() devices.Database { // CreateAccountsDB creates a new instance of the accounts database. Should only // be called once per component. func (b *BaseDendrite) CreateAccountsDB() accounts.Database { - db, err := accounts.NewDatabase(string(b.Cfg.Database.Account), b.Cfg.Matrix.ServerName) + db, err := accounts.NewDatabase(string(b.Cfg.Database.Account), b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to accounts db") } @@ -186,6 +174,7 @@ func (b *BaseDendrite) CreateAccountsDB() accounts.Database { func (b *BaseDendrite) CreateKeyDB() keydb.Database { db, err := keydb.NewDatabase( string(b.Cfg.Database.ServerKey), + b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName, b.Cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey), b.Cfg.Matrix.KeyID, @@ -256,7 +245,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { uri, err := url.Parse(string(cfg.Database.Naffka)) if err != nil || uri.Scheme == "file" { - db, err = sqlutil.Open(common.SQLiteDriverName(), string(cfg.Database.Naffka)) + db, err = sqlutil.Open(common.SQLiteDriverName(), string(cfg.Database.Naffka), nil) if err != nil { logrus.WithError(err).Panic("Failed to open naffka database") } @@ -266,7 +255,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { logrus.WithError(err).Panic("Failed to setup naffka database") } } else { - db, err = sqlutil.Open("postgres", string(cfg.Database.Naffka)) + db, err = sqlutil.Open("postgres", string(cfg.Database.Naffka), nil) if err != nil { logrus.WithError(err).Panic("Failed to open naffka database") } diff --git a/common/config/config.go b/common/config/config.go index 6b61fda7c..9a29186a6 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -188,6 +188,12 @@ type Dendrite struct { PublicRoomsAPI DataSource `yaml:"public_rooms_api"` // The Naffka database is used internally by the naffka library, if used. Naffka DataSource `yaml:"naffka,omitempty"` + // Maximum open connections to the DB (0 = use default, negative means unlimited) + MaxOpenConns int `yaml:"max_open_conns"` + // Maximum idle connections to the DB (0 = use default, negative means unlimited) + MaxIdleConns int `yaml:"max_idle_conns"` + // maximum amount of time (in seconds) a connection may be reused (<= 0 means unlimited) + ConnMaxLifetimeSec int `yaml:"conn_max_lifetime"` } `yaml:"database"` // TURN Server Config @@ -484,6 +490,15 @@ func (config *Dendrite) SetDefaults() { defaultMaxFileSizeBytes := FileSizeBytes(10485760) config.Media.MaxFileSizeBytes = &defaultMaxFileSizeBytes } + + if config.Database.MaxIdleConns == 0 { + config.Database.MaxIdleConns = 2 + } + + if config.Database.MaxOpenConns == 0 { + config.Database.MaxOpenConns = 100 + } + } // Error returns a string detailing how many errors were contained within a @@ -746,6 +761,33 @@ func (config *Dendrite) SetupTracing(serviceName string) (closer io.Closer, err ) } +// MaxIdleConns returns maximum idle connections to the DB +func (config Dendrite) MaxIdleConns() int { + return config.Database.MaxIdleConns +} + +// MaxOpenConns returns maximum open connections to the DB +func (config Dendrite) MaxOpenConns() int { + return config.Database.MaxOpenConns +} + +// ConnMaxLifetime returns maximum amount of time a connection may be reused +func (config Dendrite) ConnMaxLifetime() time.Duration { + return time.Duration(config.Database.ConnMaxLifetimeSec) * time.Second +} + +// DbProperties functions return properties used by database/sql/DB +type DbProperties interface { + MaxIdleConns() int + MaxOpenConns() int + ConnMaxLifetime() time.Duration +} + +// DbProperties returns cfg as a DbProperties interface +func (config Dendrite) DbProperties() DbProperties { + return config +} + // logrusLogger is a small wrapper that implements jaeger.Logger using logrus. type logrusLogger struct { l *logrus.Logger diff --git a/common/consumers.go b/common/consumers.go index f33993494..bc759d994 100644 --- a/common/consumers.go +++ b/common/consumers.go @@ -18,7 +18,7 @@ import ( "context" "fmt" - sarama "gopkg.in/Shopify/sarama.v1" + "github.com/Shopify/sarama" ) // A PartitionOffset is the offset into a partition of the input log. diff --git a/common/events.go b/common/events.go index 556b7b671..0c8ead3a7 100644 --- a/common/events.go +++ b/common/events.go @@ -39,13 +39,13 @@ var ErrRoomNoExists = errors.New("Room does not exist") func BuildEvent( ctx context.Context, builder *gomatrixserverlib.EventBuilder, cfg *config.Dendrite, evTime time.Time, - queryAPI api.RoomserverQueryAPI, queryRes *api.QueryLatestEventsAndStateResponse, + rsAPI api.RoomserverInternalAPI, queryRes *api.QueryLatestEventsAndStateResponse, ) (*gomatrixserverlib.Event, error) { if queryRes == nil { queryRes = &api.QueryLatestEventsAndStateResponse{} } - err := AddPrevEventsToEvent(ctx, builder, queryAPI, queryRes) + err := AddPrevEventsToEvent(ctx, builder, rsAPI, queryRes) if err != nil { // This can pass through a ErrRoomNoExists to the caller return nil, err @@ -66,7 +66,7 @@ func BuildEvent( func AddPrevEventsToEvent( ctx context.Context, builder *gomatrixserverlib.EventBuilder, - queryAPI api.RoomserverQueryAPI, queryRes *api.QueryLatestEventsAndStateResponse, + rsAPI api.RoomserverInternalAPI, queryRes *api.QueryLatestEventsAndStateResponse, ) error { eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder) if err != nil { @@ -82,8 +82,8 @@ func AddPrevEventsToEvent( RoomID: builder.RoomID, StateToFetch: eventsNeeded.Tuples(), } - if err = queryAPI.QueryLatestEventsAndState(ctx, &queryReq, queryRes); err != nil { - return fmt.Errorf("queryAPI.QueryLatestEventsAndState: %w", err) + if err = rsAPI.QueryLatestEventsAndState(ctx, &queryReq, queryRes); err != nil { + return fmt.Errorf("rsAPI.QueryLatestEventsAndState: %w", err) } if !queryRes.RoomExists { diff --git a/common/keydb/keydb.go b/common/keydb/keydb.go index fe6d87fc8..397d7849c 100644 --- a/common/keydb/keydb.go +++ b/common/keydb/keydb.go @@ -21,6 +21,7 @@ import ( "golang.org/x/crypto/ed25519" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/keydb/postgres" "github.com/matrix-org/dendrite/common/keydb/sqlite3" "github.com/matrix-org/gomatrixserverlib" @@ -29,20 +30,21 @@ import ( // NewDatabase opens a database connection. func NewDatabase( dataSourceName string, + dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName, serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, ) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID) } switch uri.Scheme { case "postgres": - return postgres.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID) case "file": return sqlite3.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID) default: - return postgres.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID) } } diff --git a/common/keydb/keydb_wasm.go b/common/keydb/keydb_wasm.go index 807ed40b4..38e595820 100644 --- a/common/keydb/keydb_wasm.go +++ b/common/keydb/keydb_wasm.go @@ -20,6 +20,7 @@ import ( "golang.org/x/crypto/ed25519" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/keydb/sqlite3" "github.com/matrix-org/gomatrixserverlib" ) @@ -27,6 +28,7 @@ import ( // NewDatabase opens a database connection. func NewDatabase( dataSourceName string, + dbProperties common.DbProperties, // nolint:unparam serverName gomatrixserverlib.ServerName, serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, diff --git a/common/keydb/postgres/keydb.go b/common/keydb/postgres/keydb.go index 2879683e0..6149d8778 100644 --- a/common/keydb/postgres/keydb.go +++ b/common/keydb/postgres/keydb.go @@ -21,6 +21,7 @@ import ( "golang.org/x/crypto/ed25519" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -37,11 +38,12 @@ type Database struct { // Returns an error if there was a problem talking to the database. func NewDatabase( dataSourceName string, + dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName, serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, ) (*Database, error) { - db, err := sqlutil.Open("postgres", dataSourceName) + db, err := sqlutil.Open("postgres", dataSourceName, dbProperties) if err != nil { return nil, err } diff --git a/common/keydb/sqlite3/keydb.go b/common/keydb/sqlite3/keydb.go index 82d2a491f..1405836a4 100644 --- a/common/keydb/sqlite3/keydb.go +++ b/common/keydb/sqlite3/keydb.go @@ -44,7 +44,7 @@ func NewDatabase( serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, ) (*Database, error) { - db, err := sqlutil.Open(common.SQLiteDriverName(), dataSourceName) + db, err := sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil) if err != nil { return nil, err } diff --git a/common/routing.go b/common/routing.go index 330912cde..cee0c162a 100644 --- a/common/routing.go +++ b/common/routing.go @@ -24,7 +24,7 @@ import ( func URLDecodeMapValues(vmap map[string]string) (map[string]string, error) { decoded := make(map[string]string, len(vmap)) for key, value := range vmap { - decodedVal, err := url.QueryUnescape(value) + decodedVal, err := url.PathUnescape(value) if err != nil { return make(map[string]string), err } diff --git a/common/sql.go b/common/sql.go index f50a58969..e93ff1c2f 100644 --- a/common/sql.go +++ b/common/sql.go @@ -18,6 +18,7 @@ import ( "database/sql" "fmt" "runtime" + "time" ) // A Transaction is something that can be committed or rolledback. @@ -99,3 +100,10 @@ func SQLiteDriverName() string { } return "sqlite3" } + +// DbProperties functions return properties used by database/sql/DB +type DbProperties interface { + MaxIdleConns() int + MaxOpenConns() int + ConnMaxLifetime() time.Duration +} diff --git a/dendrite-config.yaml b/dendrite-config.yaml index bed78a5af..8c8fba390 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -116,6 +116,9 @@ database: federation_sender: "postgres://dendrite:itsasecret@localhost/dendrite_federationsender?sslmode=disable" appservice: "postgres://dendrite:itsasecret@localhost/dendrite_appservice?sslmode=disable" public_rooms_api: "postgres://dendrite:itsasecret@localhost/dendrite_publicroomsapi?sslmode=disable" + max_open_conns: 100 + max_idle_conns: 2 + conn_max_lifetime: -1 # If using naffka you need to specify a naffka database # naffka: "postgres://dendrite:itsasecret@localhost/dendrite_naffka?sslmode=disable" diff --git a/eduserver/input/input.go b/eduserver/input/input.go index 845909452..6a4a4bb4e 100644 --- a/eduserver/input/input.go +++ b/eduserver/input/input.go @@ -18,12 +18,12 @@ import ( "net/http" "time" + "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" - "gopkg.in/Shopify/sarama.v1" ) // EDUServerInputAPI implements api.EDUServerInputAPI diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index ed96322b8..d458b8537 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -36,18 +36,16 @@ func SetupFederationAPIComponent( deviceDB devices.Database, federation *gomatrixserverlib.FederationClient, keyRing *gomatrixserverlib.KeyRing, - aliasAPI roomserverAPI.RoomserverAliasAPI, - inputAPI roomserverAPI.RoomserverInputAPI, - queryAPI roomserverAPI.RoomserverQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, - federationSenderAPI federationSenderAPI.FederationSenderQueryAPI, + federationSenderAPI federationSenderAPI.FederationSenderInternalAPI, eduProducer *producers.EDUServerProducer, ) { - roomserverProducer := producers.NewRoomserverProducer(inputAPI, queryAPI) + roomserverProducer := producers.NewRoomserverProducer(rsAPI) routing.Setup( - base.APIMux, base.Cfg, queryAPI, aliasAPI, asAPI, - roomserverProducer, eduProducer, federationSenderAPI, *keyRing, + base.APIMux, base.Cfg, rsAPI, asAPI, roomserverProducer, + eduProducer, federationSenderAPI, *keyRing, federation, accountsDB, deviceDB, ) } diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index 62471b8a9..651a4a2d2 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -33,7 +33,7 @@ import ( func Backfill( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID string, cfg *config.Dendrite, ) util.JSONResponse { @@ -69,6 +69,7 @@ func Backfill( // Populate the request. req := api.QueryBackfillRequest{ + RoomID: roomID, EarliestEventsIDs: eIDs, ServerName: request.Origin(), } @@ -81,7 +82,7 @@ func Backfill( } // Query the roomserver. - if err = query.QueryBackfill(httpReq.Context(), &req, &res); err != nil { + if err = rsAPI.QueryBackfill(httpReq.Context(), &req, &res); err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("query.QueryBackfill failed") return jsonerror.InternalServerError() } @@ -97,7 +98,10 @@ func Backfill( } var eventJSONs []json.RawMessage - for _, e := range gomatrixserverlib.ReverseTopologicalOrdering(evs) { + for _, e := range gomatrixserverlib.ReverseTopologicalOrdering( + evs, + gomatrixserverlib.TopologicalOrderByPrevEvents, + ) { eventJSONs = append(eventJSONs, e.JSON()) } diff --git a/federationapi/routing/eventauth.go b/federationapi/routing/eventauth.go index 003165c85..34eaad1c5 100644 --- a/federationapi/routing/eventauth.go +++ b/federationapi/routing/eventauth.go @@ -25,13 +25,13 @@ import ( func GetEventAuth( ctx context.Context, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID string, eventID string, ) util.JSONResponse { // TODO: Optimisation: we shouldn't be querying all the room state // that is in state.StateEvents - we just ignore it. - state, err := getState(ctx, request, query, roomID, eventID) + state, err := getState(ctx, request, rsAPI, roomID, eventID) if err != nil { return *err } diff --git a/federationapi/routing/events.go b/federationapi/routing/events.go index a91528b3d..ced9e3d53 100644 --- a/federationapi/routing/events.go +++ b/federationapi/routing/events.go @@ -16,7 +16,9 @@ package routing import ( "context" + "encoding/json" "net/http" + "time" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -27,15 +29,22 @@ import ( func GetEvent( ctx context.Context, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, eventID string, + origin gomatrixserverlib.ServerName, ) util.JSONResponse { - event, err := getEvent(ctx, request, query, eventID) + event, err := getEvent(ctx, request, rsAPI, eventID) if err != nil { return *err } - return util.JSONResponse{Code: http.StatusOK, JSON: event} + return util.JSONResponse{Code: http.StatusOK, JSON: gomatrixserverlib.Transaction{ + Origin: origin, + OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()), + PDUs: []json.RawMessage{ + event.JSON(), + }, + }} } // getEvent returns the requested event, @@ -43,11 +52,11 @@ func GetEvent( func getEvent( ctx context.Context, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, eventID string, ) (*gomatrixserverlib.Event, *util.JSONResponse) { var authResponse api.QueryServerAllowedToSeeEventResponse - err := query.QueryServerAllowedToSeeEvent( + err := rsAPI.QueryServerAllowedToSeeEvent( ctx, &api.QueryServerAllowedToSeeEventRequest{ EventID: eventID, @@ -66,7 +75,7 @@ func getEvent( } var eventsResponse api.QueryEventsByIDResponse - err = query.QueryEventsByID( + err = rsAPI.QueryEventsByID( ctx, &api.QueryEventsByIDRequest{EventIDs: []string{eventID}}, &eventsResponse, diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go index 4b367e004..064abe7e9 100644 --- a/federationapi/routing/invite.go +++ b/federationapi/routing/invite.go @@ -16,11 +16,13 @@ package routing import ( "encoding/json" + "fmt" "net/http" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/common/config" + roomserverVersion "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -44,6 +46,16 @@ func Invite( } event := inviteReq.Event() + // Check that we can accept invites for this room version. + if _, err := roomserverVersion.SupportedRoomVersion(inviteReq.RoomVersion()); err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.UnsupportedRoomVersion( + fmt.Sprintf("Room version %q is not supported by this server.", inviteReq.RoomVersion()), + ), + } + } + // Check that the room ID is correct. if event.RoomID() != roomID { return util.JSONResponse{ @@ -90,6 +102,8 @@ func Invite( httpReq.Context(), signedEvent.Headered(inviteReq.RoomVersion()), inviteReq.InviteRoomState(), + event.Origin(), + nil, ); err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("producer.SendInvite failed") return jsonerror.InternalServerError() @@ -99,6 +113,6 @@ func Invite( // the other servers in the room that we have been invited. return util.JSONResponse{ Code: http.StatusOK, - JSON: gomatrixserverlib.RespInvite{Event: signedEvent}, + JSON: gomatrixserverlib.RespInviteV2{Event: signedEvent}, } } diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index e06785954..6cadbd759 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -33,13 +33,13 @@ func MakeJoin( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, cfg *config.Dendrite, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID, userID string, remoteVersions []gomatrixserverlib.RoomVersion, ) util.JSONResponse { verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := query.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { + if err := rsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { return util.JSONResponse{ Code: http.StatusInternalServerError, JSON: jsonerror.InternalServerError(), @@ -61,9 +61,7 @@ func MakeJoin( if !remoteSupportsVersion { return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: jsonerror.UnsupportedRoomVersion( - fmt.Sprintf("Joining server does not support room version %s", verRes.RoomVersion), - ), + JSON: jsonerror.IncompatibleRoomVersion(verRes.RoomVersion), } } @@ -97,7 +95,7 @@ func MakeJoin( queryRes := api.QueryLatestEventsAndStateResponse{ RoomVersion: verRes.RoomVersion, } - event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), query, &queryRes) + event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes) if err == common.ErrRoomNoExists { return util.JSONResponse{ Code: http.StatusNotFound, @@ -132,19 +130,22 @@ func MakeJoin( } // SendJoin implements the /send_join API +// The make-join send-join dance makes much more sense as a single +// flow so the cyclomatic complexity is high: +// nolint:gocyclo func SendJoin( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, cfg *config.Dendrite, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, producer *producers.RoomserverProducer, keys gomatrixserverlib.KeyRing, roomID, eventID string, ) util.JSONResponse { verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := query.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { - util.GetLogger(httpReq.Context()).WithError(err).Error("query.QueryRoomVersionForRoom failed") + if err := rsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { + util.GetLogger(httpReq.Context()).WithError(err).Error("rsAPI.QueryRoomVersionForRoom failed") return util.JSONResponse{ Code: http.StatusInternalServerError, JSON: jsonerror.InternalServerError(), @@ -159,6 +160,16 @@ func SendJoin( } } + // Check that a state key is provided. + if event.StateKey() == nil || (event.StateKey() != nil && *event.StateKey() == "") { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON( + fmt.Sprintf("No state key was provided in the join event."), + ), + } + } + // Check that the room ID is correct. if event.RoomID() != roomID { return util.JSONResponse{ @@ -216,14 +227,14 @@ func SendJoin( // Fetch the state and auth chain. We do this before we send the events // on, in case this fails. var stateAndAuthChainResponse api.QueryStateAndAuthChainResponse - err = query.QueryStateAndAuthChain(httpReq.Context(), &api.QueryStateAndAuthChainRequest{ + err = rsAPI.QueryStateAndAuthChain(httpReq.Context(), &api.QueryStateAndAuthChainRequest{ PrevEventIDs: event.PrevEventIDs(), AuthEventIDs: event.AuthEventIDs(), RoomID: roomID, ResolveState: true, }, &stateAndAuthChainResponse) if err != nil { - util.GetLogger(httpReq.Context()).WithError(err).Error("query.QueryStateAndAuthChain failed") + util.GetLogger(httpReq.Context()).WithError(err).Error("rsAPI.QueryStateAndAuthChain failed") return jsonerror.InternalServerError() } @@ -234,27 +245,44 @@ func SendJoin( } } + // Check if the user is already in the room. If they're already in then + // there isn't much point in sending another join event into the room. + alreadyJoined := false + for _, se := range stateAndAuthChainResponse.StateEvents { + if membership, merr := se.Membership(); merr == nil { + if se.StateKey() != nil && *se.StateKey() == *event.StateKey() { + alreadyJoined = (membership == "join") + break + } + } + } + // Send the events to the room server. // We are responsible for notifying other servers that the user has joined // the room, so set SendAsServer to cfg.Matrix.ServerName - _, err = producer.SendEvents( - httpReq.Context(), - []gomatrixserverlib.HeaderedEvent{ - event.Headered(stateAndAuthChainResponse.RoomVersion), - }, - cfg.Matrix.ServerName, - nil, - ) - if err != nil { - util.GetLogger(httpReq.Context()).WithError(err).Error("producer.SendEvents failed") - return jsonerror.InternalServerError() + if !alreadyJoined { + _, err = producer.SendEvents( + httpReq.Context(), + []gomatrixserverlib.HeaderedEvent{ + event.Headered(stateAndAuthChainResponse.RoomVersion), + }, + cfg.Matrix.ServerName, + nil, + ) + if err != nil { + util.GetLogger(httpReq.Context()).WithError(err).Error("producer.SendEvents failed") + return jsonerror.InternalServerError() + } } return util.JSONResponse{ Code: http.StatusOK, - JSON: map[string]interface{}{ - "state": gomatrixserverlib.UnwrapEventHeaders(stateAndAuthChainResponse.StateEvents), - "auth_chain": gomatrixserverlib.UnwrapEventHeaders(stateAndAuthChainResponse.AuthChainEvents), + JSON: gomatrixserverlib.RespSendJoin{ + RespState: gomatrixserverlib.RespState{ + StateEvents: gomatrixserverlib.UnwrapEventHeaders(stateAndAuthChainResponse.StateEvents), + AuthEvents: gomatrixserverlib.UnwrapEventHeaders(stateAndAuthChainResponse.AuthChainEvents), + }, + Origin: cfg.Matrix.ServerName, }, } } diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index 6fc3b12ed..1124bfa27 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -30,7 +30,7 @@ func MakeLeave( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, cfg *config.Dendrite, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID, userID string, ) util.JSONResponse { _, domain, err := gomatrixserverlib.SplitID('@', userID) @@ -61,7 +61,7 @@ func MakeLeave( } var queryRes api.QueryLatestEventsAndStateResponse - event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), query, &queryRes) + event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes) if err == common.ErrRoomNoExists { return util.JSONResponse{ Code: http.StatusNotFound, @@ -102,7 +102,7 @@ func SendLeave( ) util.JSONResponse { verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := producer.QueryAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { + if err := producer.RsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UnsupportedRoomVersion(err.Error()), diff --git a/federationapi/routing/missingevents.go b/federationapi/routing/missingevents.go index 069bff3dd..ae91c5891 100644 --- a/federationapi/routing/missingevents.go +++ b/federationapi/routing/missingevents.go @@ -34,7 +34,7 @@ type getMissingEventRequest struct { func GetMissingEvents( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID string, ) util.JSONResponse { var gme getMissingEventRequest @@ -46,7 +46,7 @@ func GetMissingEvents( } var eventsResponse api.QueryMissingEventsResponse - if err := query.QueryMissingEvents( + if err := rsAPI.QueryMissingEvents( httpReq.Context(), &api.QueryMissingEventsRequest{ EarliestEvents: gme.EarliestEvents, LatestEvents: gme.LatestEvents, diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go index 7cb50e525..c58690c63 100644 --- a/federationapi/routing/query.go +++ b/federationapi/routing/query.go @@ -32,8 +32,8 @@ func RoomAliasToID( httpReq *http.Request, federation *gomatrixserverlib.FederationClient, cfg *config.Dendrite, - aliasAPI roomserverAPI.RoomserverAliasAPI, - senderAPI federationSenderAPI.FederationSenderQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, + senderAPI federationSenderAPI.FederationSenderInternalAPI, ) util.JSONResponse { roomAlias := httpReq.FormValue("room_alias") if roomAlias == "" { @@ -55,7 +55,7 @@ func RoomAliasToID( if domain == cfg.Matrix.ServerName { queryReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: roomAlias} var queryRes roomserverAPI.GetRoomIDForAliasResponse - if err = aliasAPI.GetRoomIDForAlias(httpReq.Context(), &queryReq, &queryRes); err != nil { + if err = rsAPI.GetRoomIDForAlias(httpReq.Context(), &queryReq, &queryRes); err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("aliasAPI.GetRoomIDForAlias failed") return jsonerror.InternalServerError() } diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 83bac5550..a5b8ce24e 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -44,12 +44,11 @@ const ( func Setup( apiMux *mux.Router, cfg *config.Dendrite, - query roomserverAPI.RoomserverQueryAPI, - aliasAPI roomserverAPI.RoomserverAliasAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, producer *producers.RoomserverProducer, eduProducer *producers.EDUServerProducer, - federationSenderAPI federationSenderAPI.FederationSenderQueryAPI, + federationSenderAPI federationSenderAPI.FederationSenderInternalAPI, keys gomatrixserverlib.KeyRing, federation *gomatrixserverlib.FederationClient, accountDB accounts.Database, @@ -80,7 +79,7 @@ func Setup( } return Send( httpReq, request, gomatrixserverlib.TransactionID(vars["txnID"]), - cfg, query, producer, eduProducer, keys, federation, + cfg, rsAPI, producer, eduProducer, keys, federation, ) }, )).Methods(http.MethodPut, http.MethodOptions) @@ -101,7 +100,7 @@ func Setup( v1fedmux.Handle("/3pid/onbind", common.MakeExternalAPI("3pid_onbind", func(req *http.Request) util.JSONResponse { - return CreateInvitesFrom3PIDInvites(req, query, asAPI, cfg, producer, federation, accountDB) + return CreateInvitesFrom3PIDInvites(req, rsAPI, asAPI, cfg, producer, federation, accountDB) }, )).Methods(http.MethodPost, http.MethodOptions) @@ -113,7 +112,7 @@ func Setup( return util.ErrorResponse(err) } return ExchangeThirdPartyInvite( - httpReq, request, vars["roomID"], query, cfg, federation, producer, + httpReq, request, vars["roomID"], rsAPI, cfg, federation, producer, ) }, )).Methods(http.MethodPut, http.MethodOptions) @@ -126,7 +125,7 @@ func Setup( return util.ErrorResponse(err) } return GetEvent( - httpReq.Context(), request, query, vars["eventID"], + httpReq.Context(), request, rsAPI, vars["eventID"], cfg.Matrix.ServerName, ) }, )).Methods(http.MethodGet) @@ -139,7 +138,7 @@ func Setup( return util.ErrorResponse(err) } return GetState( - httpReq.Context(), request, query, vars["roomID"], + httpReq.Context(), request, rsAPI, vars["roomID"], ) }, )).Methods(http.MethodGet) @@ -152,7 +151,7 @@ func Setup( return util.ErrorResponse(err) } return GetStateIDs( - httpReq.Context(), request, query, vars["roomID"], + httpReq.Context(), request, rsAPI, vars["roomID"], ) }, )).Methods(http.MethodGet) @@ -162,7 +161,7 @@ func Setup( func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars := mux.Vars(httpReq) return GetEventAuth( - httpReq.Context(), request, query, vars["roomID"], vars["eventID"], + httpReq.Context(), request, rsAPI, vars["roomID"], vars["eventID"], ) }, )).Methods(http.MethodGet) @@ -171,7 +170,7 @@ func Setup( "federation_query_room_alias", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { return RoomAliasToID( - httpReq, federation, cfg, aliasAPI, federationSenderAPI, + httpReq, federation, cfg, rsAPI, federationSenderAPI, ) }, )).Methods(http.MethodGet) @@ -222,7 +221,7 @@ func Setup( remoteVersions = append(remoteVersions, gomatrixserverlib.RoomVersionV1) } return MakeJoin( - httpReq, request, cfg, query, roomID, eventID, remoteVersions, + httpReq, request, cfg, rsAPI, roomID, eventID, remoteVersions, ) }, )).Methods(http.MethodGet) @@ -237,7 +236,7 @@ func Setup( roomID := vars["roomID"] eventID := vars["eventID"] return SendJoin( - httpReq, request, cfg, query, producer, keys, roomID, eventID, + httpReq, request, cfg, rsAPI, producer, keys, roomID, eventID, ) }, )).Methods(http.MethodPut) @@ -252,7 +251,7 @@ func Setup( roomID := vars["roomID"] eventID := vars["eventID"] return MakeLeave( - httpReq, request, cfg, query, roomID, eventID, + httpReq, request, cfg, rsAPI, roomID, eventID, ) }, )).Methods(http.MethodGet) @@ -286,7 +285,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetMissingEvents(httpReq, request, query, vars["roomID"]) + return GetMissingEvents(httpReq, request, rsAPI, vars["roomID"]) }, )).Methods(http.MethodPost) @@ -297,7 +296,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return Backfill(httpReq, request, query, vars["roomID"], cfg) + return Backfill(httpReq, request, rsAPI, vars["roomID"], cfg) }, )).Methods(http.MethodGet) } diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 5a9766f81..88411b81a 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -34,7 +34,7 @@ func Send( request *gomatrixserverlib.FederationRequest, txnID gomatrixserverlib.TransactionID, cfg *config.Dendrite, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, producer *producers.RoomserverProducer, eduProducer *producers.EDUServerProducer, keys gomatrixserverlib.KeyRing, @@ -42,7 +42,7 @@ func Send( ) util.JSONResponse { t := txnReq{ context: httpReq.Context(), - query: query, + rsAPI: rsAPI, producer: producer, eduProducer: eduProducer, keys: keys, @@ -99,7 +99,7 @@ func Send( type txnReq struct { gomatrixserverlib.Transaction context context.Context - query api.RoomserverQueryAPI + rsAPI api.RoomserverInternalAPI producer *producers.RoomserverProducer eduProducer *producers.EDUServerProducer keys gomatrixserverlib.KeyRing @@ -120,7 +120,7 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { } verReq := api.QueryRoomVersionForRoomRequest{RoomID: header.RoomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := t.query.QueryRoomVersionForRoom(t.context, &verReq, &verRes); err != nil { + if err := t.rsAPI.QueryRoomVersionForRoom(t.context, &verReq, &verRes); err != nil { util.GetLogger(t.context).WithError(err).Warn("Transaction: Failed to query room version for room", verReq.RoomID) return nil, roomNotFoundError{verReq.RoomID} } @@ -228,7 +228,7 @@ func (t *txnReq) processEvent(e gomatrixserverlib.Event) error { StateToFetch: needed.Tuples(), } var stateResp api.QueryStateAfterEventsResponse - if err := t.query.QueryStateAfterEvents(t.context, &stateReq, &stateResp); err != nil { + if err := t.rsAPI.QueryStateAfterEvents(t.context, &stateReq, &stateResp); err != nil { return err } diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go index 548598dd7..f90c494c3 100644 --- a/federationapi/routing/state.go +++ b/federationapi/routing/state.go @@ -27,7 +27,7 @@ import ( func GetState( ctx context.Context, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID string, ) util.JSONResponse { eventID, err := parseEventIDParam(request) @@ -35,7 +35,7 @@ func GetState( return *err } - state, err := getState(ctx, request, query, roomID, eventID) + state, err := getState(ctx, request, rsAPI, roomID, eventID) if err != nil { return *err } @@ -47,7 +47,7 @@ func GetState( func GetStateIDs( ctx context.Context, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID string, ) util.JSONResponse { eventID, err := parseEventIDParam(request) @@ -55,7 +55,7 @@ func GetStateIDs( return *err } - state, err := getState(ctx, request, query, roomID, eventID) + state, err := getState(ctx, request, rsAPI, roomID, eventID) if err != nil { return *err } @@ -94,11 +94,11 @@ func parseEventIDParam( func getState( ctx context.Context, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID string, eventID string, ) (*gomatrixserverlib.RespState, *util.JSONResponse) { - event, resErr := getEvent(ctx, request, query, eventID) + event, resErr := getEvent(ctx, request, rsAPI, eventID) if resErr != nil { return nil, resErr } @@ -110,7 +110,7 @@ func getState( authEventIDs := getIDsFromEventRef(event.AuthEvents()) var response api.QueryStateAndAuthChainResponse - err := query.QueryStateAndAuthChain( + err := rsAPI.QueryStateAndAuthChain( ctx, &api.QueryStateAndAuthChainRequest{ RoomID: roomID, diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index f93d934ed..3c1d09e11 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -58,7 +58,7 @@ var ( // CreateInvitesFrom3PIDInvites implements POST /_matrix/federation/v1/3pid/onbind func CreateInvitesFrom3PIDInvites( - req *http.Request, queryAPI roomserverAPI.RoomserverQueryAPI, + req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, cfg *config.Dendrite, producer *producers.RoomserverProducer, federation *gomatrixserverlib.FederationClient, accountDB accounts.Database, @@ -72,7 +72,7 @@ func CreateInvitesFrom3PIDInvites( for _, inv := range body.Invites { verReq := api.QueryRoomVersionForRoomRequest{RoomID: inv.RoomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := queryAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { + if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UnsupportedRoomVersion(err.Error()), @@ -80,7 +80,7 @@ func CreateInvitesFrom3PIDInvites( } event, err := createInviteFrom3PIDInvite( - req.Context(), queryAPI, asAPI, cfg, inv, federation, accountDB, + req.Context(), rsAPI, asAPI, cfg, inv, federation, accountDB, ) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("createInviteFrom3PIDInvite failed") @@ -108,7 +108,7 @@ func ExchangeThirdPartyInvite( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, roomID string, - queryAPI roomserverAPI.RoomserverQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, cfg *config.Dendrite, federation *gomatrixserverlib.FederationClient, producer *producers.RoomserverProducer, @@ -148,7 +148,7 @@ func ExchangeThirdPartyInvite( verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err = queryAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { + if err = rsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UnsupportedRoomVersion(err.Error()), @@ -156,7 +156,7 @@ func ExchangeThirdPartyInvite( } // Auth and build the event from what the remote server sent us - event, err := buildMembershipEvent(httpReq.Context(), &builder, queryAPI, cfg) + event, err := buildMembershipEvent(httpReq.Context(), &builder, rsAPI, cfg) if err == errNotInRoom { return util.JSONResponse{ Code: http.StatusNotFound, @@ -199,14 +199,14 @@ func ExchangeThirdPartyInvite( // Returns an error if there was a problem building the event or fetching the // necessary data to do so. func createInviteFrom3PIDInvite( - ctx context.Context, queryAPI roomserverAPI.RoomserverQueryAPI, + ctx context.Context, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, cfg *config.Dendrite, inv invite, federation *gomatrixserverlib.FederationClient, accountDB accounts.Database, ) (*gomatrixserverlib.Event, error) { verReq := api.QueryRoomVersionForRoomRequest{RoomID: inv.RoomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := queryAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil { + if err := rsAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil { return nil, err } @@ -245,7 +245,7 @@ func createInviteFrom3PIDInvite( return nil, err } - event, err := buildMembershipEvent(ctx, builder, queryAPI, cfg) + event, err := buildMembershipEvent(ctx, builder, rsAPI, cfg) if err == errNotInRoom { return nil, sendToRemoteServer(ctx, inv, federation, cfg, *builder) } @@ -263,7 +263,7 @@ func createInviteFrom3PIDInvite( // Returns an error if something failed during the process. func buildMembershipEvent( ctx context.Context, - builder *gomatrixserverlib.EventBuilder, queryAPI roomserverAPI.RoomserverQueryAPI, + builder *gomatrixserverlib.EventBuilder, rsAPI roomserverAPI.RoomserverInternalAPI, cfg *config.Dendrite, ) (*gomatrixserverlib.Event, error) { eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder) @@ -281,7 +281,7 @@ func buildMembershipEvent( StateToFetch: eventsNeeded.Tuples(), } var queryRes roomserverAPI.QueryLatestEventsAndStateResponse - if err = queryAPI.QueryLatestEventsAndState(ctx, &queryReq, &queryRes); err != nil { + if err = rsAPI.QueryLatestEventsAndState(ctx, &queryReq, &queryRes); err != nil { return nil, err } diff --git a/federationsender/api/api.go b/federationsender/api/api.go new file mode 100644 index 000000000..678f02e68 --- /dev/null +++ b/federationsender/api/api.go @@ -0,0 +1,59 @@ +package api + +import ( + "context" + "errors" + "net/http" +) + +// FederationSenderInternalAPI is used to query information from the federation sender. +type FederationSenderInternalAPI interface { + // PerformDirectoryLookup looks up a remote room ID from a room alias. + PerformDirectoryLookup( + ctx context.Context, + request *PerformDirectoryLookupRequest, + response *PerformDirectoryLookupResponse, + ) error + // Query the joined hosts and the membership events accounting for their participation in a room. + // Note that if a server has multiple users in the room, it will have multiple entries in the returned slice. + // See `QueryJoinedHostServerNamesInRoom` for a de-duplicated version. + QueryJoinedHostsInRoom( + ctx context.Context, + request *QueryJoinedHostsInRoomRequest, + response *QueryJoinedHostsInRoomResponse, + ) error + // Query the server names of the joined hosts in a room. + // Unlike QueryJoinedHostsInRoom, this function returns a de-duplicated slice + // containing only the server names (without information for membership events). + QueryJoinedHostServerNamesInRoom( + ctx context.Context, + request *QueryJoinedHostServerNamesInRoomRequest, + response *QueryJoinedHostServerNamesInRoomResponse, + ) error + // Handle an instruction to make_join & send_join with a remote server. + PerformJoin( + ctx context.Context, + request *PerformJoinRequest, + response *PerformJoinResponse, + ) error + // Handle an instruction to make_leave & send_leave with a remote server. + PerformLeave( + ctx context.Context, + request *PerformLeaveRequest, + response *PerformLeaveResponse, + ) error +} + +// NewFederationSenderInternalAPIHTTP creates a FederationSenderInternalAPI implemented by talking to a HTTP POST API. +// If httpClient is nil an error is returned +func NewFederationSenderInternalAPIHTTP(federationSenderURL string, httpClient *http.Client) (FederationSenderInternalAPI, error) { + if httpClient == nil { + return nil, errors.New("NewFederationSenderInternalAPIHTTP: httpClient is ") + } + return &httpFederationSenderInternalAPI{federationSenderURL, httpClient}, nil +} + +type httpFederationSenderInternalAPI struct { + federationSenderURL string + httpClient *http.Client +} diff --git a/federationsender/api/perform.go b/federationsender/api/perform.go new file mode 100644 index 000000000..a7b12adc7 --- /dev/null +++ b/federationsender/api/perform.go @@ -0,0 +1,87 @@ +package api + +import ( + "context" + + commonHTTP "github.com/matrix-org/dendrite/common/http" + "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/gomatrixserverlib" + "github.com/opentracing/opentracing-go" +) + +const ( + // FederationSenderPerformJoinRequestPath is the HTTP path for the PerformJoinRequest API. + FederationSenderPerformDirectoryLookupRequestPath = "/api/federationsender/performDirectoryLookup" + + // FederationSenderPerformJoinRequestPath is the HTTP path for the PerformJoinRequest API. + FederationSenderPerformJoinRequestPath = "/api/federationsender/performJoinRequest" + + // FederationSenderPerformLeaveRequestPath is the HTTP path for the PerformLeaveRequest API. + FederationSenderPerformLeaveRequestPath = "/api/federationsender/performLeaveRequest" +) + +type PerformDirectoryLookupRequest struct { + RoomAlias string `json:"room_alias"` + ServerName gomatrixserverlib.ServerName `json:"server_name"` +} + +type PerformDirectoryLookupResponse struct { + RoomID string `json:"room_id"` + ServerNames []gomatrixserverlib.ServerName `json:"server_names"` +} + +// Handle an instruction to make_join & send_join with a remote server. +func (h *httpFederationSenderInternalAPI) PerformDirectoryLookup( + ctx context.Context, + request *PerformDirectoryLookupRequest, + response *PerformDirectoryLookupResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformDirectoryLookup") + defer span.Finish() + + apiURL := h.federationSenderURL + FederationSenderPerformDirectoryLookupRequestPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +type PerformJoinRequest struct { + RoomID string `json:"room_id"` + UserID string `json:"user_id"` + ServerNames types.ServerNames `json:"server_names"` + Content map[string]interface{} `json:"content"` +} + +type PerformJoinResponse struct { +} + +// Handle an instruction to make_join & send_join with a remote server. +func (h *httpFederationSenderInternalAPI) PerformJoin( + ctx context.Context, + request *PerformJoinRequest, + response *PerformJoinResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformJoinRequest") + defer span.Finish() + + apiURL := h.federationSenderURL + FederationSenderPerformJoinRequestPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +type PerformLeaveRequest struct { + RoomID string `json:"room_id"` +} + +type PerformLeaveResponse struct { +} + +// Handle an instruction to make_leave & send_leave with a remote server. +func (h *httpFederationSenderInternalAPI) PerformLeave( + ctx context.Context, + request *PerformLeaveRequest, + response *PerformLeaveResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformLeaveRequest") + defer span.Finish() + + apiURL := h.federationSenderURL + FederationSenderPerformLeaveRequestPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} diff --git a/federationsender/api/query.go b/federationsender/api/query.go index 7c0ca7ff2..7a58cc863 100644 --- a/federationsender/api/query.go +++ b/federationsender/api/query.go @@ -2,16 +2,20 @@ package api import ( "context" - "errors" - "net/http" commonHTTP "github.com/matrix-org/dendrite/common/http" + "github.com/matrix-org/dendrite/federationsender/types" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/dendrite/federationsender/types" "github.com/opentracing/opentracing-go" ) +// FederationSenderQueryJoinedHostsInRoomPath is the HTTP path for the QueryJoinedHostsInRoom API. +const FederationSenderQueryJoinedHostsInRoomPath = "/api/federationsender/queryJoinedHostsInRoom" + +// FederationSenderQueryJoinedHostServerNamesInRoomPath is the HTTP path for the QueryJoinedHostServerNamesInRoom API. +const FederationSenderQueryJoinedHostServerNamesInRoomPath = "/api/federationsender/queryJoinedHostServerNamesInRoom" + // QueryJoinedHostsInRoomRequest is a request to QueryJoinedHostsInRoom type QueryJoinedHostsInRoomRequest struct { RoomID string `json:"room_id"` @@ -22,6 +26,19 @@ type QueryJoinedHostsInRoomResponse struct { JoinedHosts []types.JoinedHost `json:"joined_hosts"` } +// QueryJoinedHostsInRoom implements FederationSenderInternalAPI +func (h *httpFederationSenderInternalAPI) QueryJoinedHostsInRoom( + ctx context.Context, + request *QueryJoinedHostsInRoomRequest, + response *QueryJoinedHostsInRoomResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostsInRoom") + defer span.Finish() + + apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostsInRoomPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + // QueryJoinedHostServerNamesRequest is a request to QueryJoinedHostServerNames type QueryJoinedHostServerNamesInRoomRequest struct { RoomID string `json:"room_id"` @@ -32,61 +49,8 @@ type QueryJoinedHostServerNamesInRoomResponse struct { ServerNames []gomatrixserverlib.ServerName `json:"server_names"` } -// FederationSenderQueryAPI is used to query information from the federation sender. -type FederationSenderQueryAPI interface { - // Query the joined hosts and the membership events accounting for their participation in a room. - // Note that if a server has multiple users in the room, it will have multiple entries in the returned slice. - // See `QueryJoinedHostServerNamesInRoom` for a de-duplicated version. - QueryJoinedHostsInRoom( - ctx context.Context, - request *QueryJoinedHostsInRoomRequest, - response *QueryJoinedHostsInRoomResponse, - ) error - // Query the server names of the joined hosts in a room. - // Unlike QueryJoinedHostsInRoom, this function returns a de-duplicated slice - // containing only the server names (without information for membership events). - QueryJoinedHostServerNamesInRoom( - ctx context.Context, - request *QueryJoinedHostServerNamesInRoomRequest, - response *QueryJoinedHostServerNamesInRoomResponse, - ) error -} - -// FederationSenderQueryJoinedHostsInRoomPath is the HTTP path for the QueryJoinedHostsInRoom API. -const FederationSenderQueryJoinedHostsInRoomPath = "/api/federationsender/queryJoinedHostsInRoom" - -// FederationSenderQueryJoinedHostServerNamesInRoomPath is the HTTP path for the QueryJoinedHostServerNamesInRoom API. -const FederationSenderQueryJoinedHostServerNamesInRoomPath = "/api/federationsender/queryJoinedHostServerNamesInRoom" - -// NewFederationSenderQueryAPIHTTP creates a FederationSenderQueryAPI implemented by talking to a HTTP POST API. -// If httpClient is nil an error is returned -func NewFederationSenderQueryAPIHTTP(federationSenderURL string, httpClient *http.Client) (FederationSenderQueryAPI, error) { - if httpClient == nil { - return nil, errors.New("NewFederationSenderQueryAPIHTTP: httpClient is ") - } - return &httpFederationSenderQueryAPI{federationSenderURL, httpClient}, nil -} - -type httpFederationSenderQueryAPI struct { - federationSenderURL string - httpClient *http.Client -} - -// QueryJoinedHostsInRoom implements FederationSenderQueryAPI -func (h *httpFederationSenderQueryAPI) QueryJoinedHostsInRoom( - ctx context.Context, - request *QueryJoinedHostsInRoomRequest, - response *QueryJoinedHostsInRoomResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostsInRoom") - defer span.Finish() - - apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostsInRoomPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// QueryJoinedHostServerNamesInRoom implements FederationSenderQueryAPI -func (h *httpFederationSenderQueryAPI) QueryJoinedHostServerNamesInRoom( +// QueryJoinedHostServerNamesInRoom implements FederationSenderInternalAPI +func (h *httpFederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom( ctx context.Context, request *QueryJoinedHostServerNamesInRoomRequest, response *QueryJoinedHostServerNamesInRoomResponse, diff --git a/federationsender/consumers/eduserver.go b/federationsender/consumers/eduserver.go index 4d2445f3c..269701d77 100644 --- a/federationsender/consumers/eduserver.go +++ b/federationsender/consumers/eduserver.go @@ -16,6 +16,7 @@ import ( "context" "encoding/json" + "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/eduserver/api" @@ -23,7 +24,6 @@ import ( "github.com/matrix-org/dendrite/federationsender/storage" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" - "gopkg.in/Shopify/sarama.v1" ) // OutputTypingEventConsumer consumes events that originate in EDU server. diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index f59405af0..67d08b339 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -19,6 +19,7 @@ import ( "encoding/json" "fmt" + "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/federationsender/queue" @@ -27,16 +28,16 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" - sarama "gopkg.in/Shopify/sarama.v1" + "github.com/tidwall/gjson" ) // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { - cfg *config.Dendrite - roomServerConsumer *common.ContinualConsumer - db storage.Database - queues *queue.OutgoingQueues - query api.RoomserverQueryAPI + cfg *config.Dendrite + rsAPI api.RoomserverInternalAPI + rsConsumer *common.ContinualConsumer + db storage.Database + queues *queue.OutgoingQueues } // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. @@ -45,7 +46,7 @@ func NewOutputRoomEventConsumer( kafkaConsumer sarama.Consumer, queues *queue.OutgoingQueues, store storage.Database, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, ) *OutputRoomEventConsumer { consumer := common.ContinualConsumer{ Topic: string(cfg.Kafka.Topics.OutputRoomEvent), @@ -53,11 +54,11 @@ func NewOutputRoomEventConsumer( PartitionStore: store, } s := &OutputRoomEventConsumer{ - cfg: cfg, - roomServerConsumer: &consumer, - db: store, - queues: queues, - query: queryAPI, + cfg: cfg, + rsConsumer: &consumer, + db: store, + queues: queues, + rsAPI: rsAPI, } consumer.ProcessMessage = s.onMessage @@ -66,7 +67,7 @@ func NewOutputRoomEventConsumer( // Start consuming from room servers func (s *OutputRoomEventConsumer) Start() error { - return s.roomServerConsumer.Start() + return s.rsConsumer.Start() } // onMessage is called when the federation server receives a new event from the room server output log. @@ -187,49 +188,12 @@ func (s *OutputRoomEventConsumer) processInvite(oie api.OutputNewInviteEvent) er return nil } - // When sending a v2 invite, the inviting server should try and include - // a "stripped down" version of the room state. This is pretty much just - // enough information for the remote side to show something useful to the - // user, like the room name, aliases etc. + // Try to extract the room invite state. The roomserver will have stashed + // this for us in invite_room_state if it didn't already exist. strippedState := []gomatrixserverlib.InviteV2StrippedState{} - stateWanted := []string{ - gomatrixserverlib.MRoomName, gomatrixserverlib.MRoomCanonicalAlias, - gomatrixserverlib.MRoomAliases, gomatrixserverlib.MRoomJoinRules, - } - - // For each of the state keys that we want to try and send, ask the - // roomserver if we have a state event for that room that matches the - // state key. - for _, wanted := range stateWanted { - queryReq := api.QueryLatestEventsAndStateRequest{ - RoomID: oie.Event.RoomID(), - StateToFetch: []gomatrixserverlib.StateKeyTuple{ - gomatrixserverlib.StateKeyTuple{ - EventType: wanted, - StateKey: "", - }, - }, - } - // If this fails then we just move onto the next event - we don't - // actually know at this point whether the room even has that type - // of state. - queryRes := api.QueryLatestEventsAndStateResponse{} - if err := s.query.QueryLatestEventsAndState(context.TODO(), &queryReq, &queryRes); err != nil { - log.WithFields(log.Fields{ - "room_id": queryReq.RoomID, - "event_type": wanted, - }).WithError(err).Info("couldn't find state to strip") - continue - } - // Append the stripped down copy of the state to our list. - for _, headeredEvent := range queryRes.StateEvents { - event := headeredEvent.Unwrap() - strippedState = append(strippedState, gomatrixserverlib.NewInviteV2StrippedState(&event)) - - log.WithFields(log.Fields{ - "room_id": queryReq.RoomID, - "event_type": event.Type(), - }).Info("adding stripped state") + if inviteRoomState := gjson.GetBytes(oie.Event.Unsigned(), "invite_room_state"); inviteRoomState.Exists() { + if err := json.Unmarshal([]byte(inviteRoomState.Raw), &strippedState); err != nil { + log.WithError(err).Warn("failed to extract invite_room_state from event unsigned") } } @@ -405,7 +369,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( // from the roomserver using the query API. eventReq := api.QueryEventsByIDRequest{EventIDs: missing} var eventResp api.QueryEventsByIDResponse - if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { + if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { return nil, err } diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index a318d2099..cf4395527 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -20,7 +20,8 @@ import ( "github.com/matrix-org/dendrite/common/basecomponent" "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/federationsender/consumers" - "github.com/matrix-org/dendrite/federationsender/query" + "github.com/matrix-org/dendrite/federationsender/internal" + "github.com/matrix-org/dendrite/federationsender/producers" "github.com/matrix-org/dendrite/federationsender/queue" "github.com/matrix-org/dendrite/federationsender/storage" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" @@ -33,18 +34,21 @@ import ( func SetupFederationSenderComponent( base *basecomponent.BaseDendrite, federation *gomatrixserverlib.FederationClient, - rsQueryAPI roomserverAPI.RoomserverQueryAPI, -) api.FederationSenderQueryAPI { - federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender)) + rsAPI roomserverAPI.RoomserverInternalAPI, + keyRing *gomatrixserverlib.KeyRing, +) api.FederationSenderInternalAPI { + federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender), base.Cfg.DbProperties()) if err != nil { logrus.WithError(err).Panic("failed to connect to federation sender db") } - queues := queue.NewOutgoingQueues(base.Cfg.Matrix.ServerName, federation) + roomserverProducer := producers.NewRoomserverProducer(rsAPI, base.Cfg.Matrix.ServerName) + + queues := queue.NewOutgoingQueues(base.Cfg.Matrix.ServerName, federation, roomserverProducer) rsConsumer := consumers.NewOutputRoomEventConsumer( base.Cfg, base.KafkaConsumer, queues, - federationSenderDB, rsQueryAPI, + federationSenderDB, rsAPI, ) if err = rsConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start room server consumer") @@ -57,10 +61,10 @@ func SetupFederationSenderComponent( logrus.WithError(err).Panic("failed to start typing server consumer") } - queryAPI := query.FederationSenderQueryAPI{ - DB: federationSenderDB, - } + queryAPI := internal.NewFederationSenderInternalAPI( + federationSenderDB, base.Cfg, roomserverProducer, federation, keyRing, + ) queryAPI.SetupHTTP(http.DefaultServeMux) - return &queryAPI + return queryAPI } diff --git a/federationsender/internal/api.go b/federationsender/internal/api.go new file mode 100644 index 000000000..89a1fda40 --- /dev/null +++ b/federationsender/internal/api.go @@ -0,0 +1,97 @@ +package internal + +import ( + "encoding/json" + "net/http" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/federationsender/producers" + "github.com/matrix-org/dendrite/federationsender/storage" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" +) + +// FederationSenderInternalAPI is an implementation of api.FederationSenderInternalAPI +type FederationSenderInternalAPI struct { + api.FederationSenderInternalAPI + db storage.Database + cfg *config.Dendrite + producer *producers.RoomserverProducer + federation *gomatrixserverlib.FederationClient + keyRing *gomatrixserverlib.KeyRing +} + +func NewFederationSenderInternalAPI( + db storage.Database, cfg *config.Dendrite, + producer *producers.RoomserverProducer, + federation *gomatrixserverlib.FederationClient, + keyRing *gomatrixserverlib.KeyRing, +) *FederationSenderInternalAPI { + return &FederationSenderInternalAPI{ + db: db, + cfg: cfg, + producer: producer, + federation: federation, + keyRing: keyRing, + } +} + +// SetupHTTP adds the FederationSenderInternalAPI handlers to the http.ServeMux. +func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) { + servMux.Handle( + api.FederationSenderQueryJoinedHostsInRoomPath, + common.MakeInternalAPI("QueryJoinedHostsInRoom", func(req *http.Request) util.JSONResponse { + var request api.QueryJoinedHostsInRoomRequest + var response api.QueryJoinedHostsInRoomResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := f.QueryJoinedHostsInRoom(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.FederationSenderQueryJoinedHostServerNamesInRoomPath, + common.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse { + var request api.QueryJoinedHostServerNamesInRoomRequest + var response api.QueryJoinedHostServerNamesInRoomResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := f.QueryJoinedHostServerNamesInRoom(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle(api.FederationSenderPerformJoinRequestPath, + common.MakeInternalAPI("PerformJoinRequest", func(req *http.Request) util.JSONResponse { + var request api.PerformJoinRequest + var response api.PerformJoinResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := f.PerformJoin(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle(api.FederationSenderPerformLeaveRequestPath, + common.MakeInternalAPI("PerformLeaveRequest", func(req *http.Request) util.JSONResponse { + var request api.PerformLeaveRequest + var response api.PerformLeaveResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := f.PerformLeave(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) +} diff --git a/federationsender/internal/perform.go b/federationsender/internal/perform.go new file mode 100644 index 000000000..161b689e1 --- /dev/null +++ b/federationsender/internal/perform.go @@ -0,0 +1,157 @@ +package internal + +import ( + "context" + "fmt" + "time" + + "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/federationsender/internal/perform" + "github.com/matrix-org/dendrite/roomserver/version" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/sirupsen/logrus" +) + +// PerformLeaveRequest implements api.FederationSenderInternalAPI +func (r *FederationSenderInternalAPI) PerformDirectoryLookup( + ctx context.Context, + request *api.PerformDirectoryLookupRequest, + response *api.PerformDirectoryLookupResponse, +) (err error) { + dir, err := r.federation.LookupRoomAlias( + ctx, + request.ServerName, + request.RoomAlias, + ) + if err != nil { + return err + } + response.RoomID = dir.RoomID + response.ServerNames = dir.Servers + return nil +} + +// PerformJoinRequest implements api.FederationSenderInternalAPI +func (r *FederationSenderInternalAPI) PerformJoin( + ctx context.Context, + request *api.PerformJoinRequest, + response *api.PerformJoinResponse, +) (err error) { + // Look up the supported room versions. + var supportedVersions []gomatrixserverlib.RoomVersion + for version := range version.SupportedRoomVersions() { + supportedVersions = append(supportedVersions, version) + } + + // Deduplicate the server names we were provided. + util.Unique(request.ServerNames) + + // Try each server that we were provided until we land on one that + // successfully completes the make-join send-join dance. + for _, serverName := range request.ServerNames { + // Try to perform a make_join using the information supplied in the + // request. + respMakeJoin, err := r.federation.MakeJoin( + ctx, + serverName, + request.RoomID, + request.UserID, + supportedVersions, + ) + if err != nil { + // TODO: Check if the user was not allowed to join the room. + return fmt.Errorf("r.federation.MakeJoin: %w", err) + } + + // Set all the fields to be what they should be, this should be a no-op + // but it's possible that the remote server returned us something "odd" + respMakeJoin.JoinEvent.Type = gomatrixserverlib.MRoomMember + respMakeJoin.JoinEvent.Sender = request.UserID + respMakeJoin.JoinEvent.StateKey = &request.UserID + respMakeJoin.JoinEvent.RoomID = request.RoomID + respMakeJoin.JoinEvent.Redacts = "" + if request.Content == nil { + request.Content = map[string]interface{}{} + } + request.Content["membership"] = "join" + if err = respMakeJoin.JoinEvent.SetContent(request.Content); err != nil { + return fmt.Errorf("respMakeJoin.JoinEvent.SetContent: %w", err) + } + if err = respMakeJoin.JoinEvent.SetUnsigned(struct{}{}); err != nil { + return fmt.Errorf("respMakeJoin.JoinEvent.SetUnsigned: %w", err) + } + + // Work out if we support the room version that has been supplied in + // the make_join response. + if respMakeJoin.RoomVersion == "" { + respMakeJoin.RoomVersion = gomatrixserverlib.RoomVersionV1 + } + if _, err = respMakeJoin.RoomVersion.EventFormat(); err != nil { + return fmt.Errorf("respMakeJoin.RoomVersion.EventFormat: %w", err) + } + + // Build the join event. + event, err := respMakeJoin.JoinEvent.Build( + time.Now(), + r.cfg.Matrix.ServerName, + r.cfg.Matrix.KeyID, + r.cfg.Matrix.PrivateKey, + respMakeJoin.RoomVersion, + ) + if err != nil { + return fmt.Errorf("respMakeJoin.JoinEvent.Build: %w", err) + } + + // Try to perform a send_join using the newly built event. + respSendJoin, err := r.federation.SendJoin( + ctx, + serverName, + event, + respMakeJoin.RoomVersion, + ) + if err != nil { + logrus.WithError(err).Warnf("r.federation.SendJoin failed") + continue + } + + // Check that the send_join response was valid. + joinCtx := perform.JoinContext(r.federation, r.keyRing) + if err = joinCtx.CheckSendJoinResponse( + ctx, event, serverName, respMakeJoin, respSendJoin, + ); err != nil { + logrus.WithError(err).Warnf("joinCtx.CheckSendJoinResponse failed") + continue + } + + // If we successfully performed a send_join above then the other + // server now thinks we're a part of the room. Send the newly + // returned state to the roomserver to update our local view. + if err = r.producer.SendEventWithState( + ctx, + respSendJoin.ToRespState(), + event.Headered(respMakeJoin.RoomVersion), + ); err != nil { + logrus.WithError(err).Warnf("r.producer.SendEventWithState failed") + continue + } + + // We're all good. + return nil + } + + // If we reach here then we didn't complete a join for some reason. + return fmt.Errorf( + "failed to join user %q to room %q through %d server(s)", + request.UserID, request.RoomID, len(request.ServerNames), + ) +} + +// PerformLeaveRequest implements api.FederationSenderInternalAPI +func (r *FederationSenderInternalAPI) PerformLeave( + ctx context.Context, + request *api.PerformLeaveRequest, + response *api.PerformLeaveResponse, +) (err error) { + return nil +} diff --git a/federationsender/internal/perform/join.go b/federationsender/internal/perform/join.go new file mode 100644 index 000000000..3c7ef0765 --- /dev/null +++ b/federationsender/internal/perform/join.go @@ -0,0 +1,70 @@ +package perform + +import ( + "context" + "fmt" + + "github.com/matrix-org/gomatrixserverlib" +) + +// This file contains helpers for the PerformJoin function. + +type joinContext struct { + federation *gomatrixserverlib.FederationClient + keyRing *gomatrixserverlib.KeyRing +} + +// Returns a new join context. +func JoinContext(f *gomatrixserverlib.FederationClient, k *gomatrixserverlib.KeyRing) *joinContext { + return &joinContext{ + federation: f, + keyRing: k, + } +} + +// checkSendJoinResponse checks that all of the signatures are correct +// and that the join is allowed by the supplied state. +func (r joinContext) CheckSendJoinResponse( + ctx context.Context, + event gomatrixserverlib.Event, + server gomatrixserverlib.ServerName, + respMakeJoin gomatrixserverlib.RespMakeJoin, + respSendJoin gomatrixserverlib.RespSendJoin, +) error { + // A list of events that we have retried, if they were not included in + // the auth events supplied in the send_join. + retries := map[string]bool{} + +retryCheck: + // TODO: Can we expand Check here to return a list of missing auth + // events rather than failing one at a time? + if err := respSendJoin.Check(ctx, r.keyRing, event); err != nil { + switch e := err.(type) { + case gomatrixserverlib.MissingAuthEventError: + // Check that we haven't already retried for this event, prevents + // us from ending up in endless loops + if !retries[e.AuthEventID] { + // Ask the server that we're talking to right now for the event + tx, txerr := r.federation.GetEvent(ctx, server, e.AuthEventID) + if txerr != nil { + return fmt.Errorf("r.federation.GetEvent: %w", txerr) + } + // For each event returned, add it to the auth events. + for _, pdu := range tx.PDUs { + ev, everr := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, respMakeJoin.RoomVersion) + if everr != nil { + return fmt.Errorf("gomatrixserverlib.NewEventFromUntrustedJSON: %w", everr) + } + respSendJoin.AuthEvents = append(respSendJoin.AuthEvents, ev) + } + // Mark the event as retried and then give the check another go. + retries[e.AuthEventID] = true + goto retryCheck + } + return fmt.Errorf("respSendJoin (after retries): %w", e) + default: + return fmt.Errorf("respSendJoin: %w", err) + } + } + return nil +} diff --git a/federationsender/internal/query.go b/federationsender/internal/query.go new file mode 100644 index 000000000..88dd50a62 --- /dev/null +++ b/federationsender/internal/query.go @@ -0,0 +1,39 @@ +package internal + +import ( + "context" + + "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/gomatrixserverlib" +) + +// QueryJoinedHostsInRoom implements api.FederationSenderInternalAPI +func (f *FederationSenderInternalAPI) QueryJoinedHostsInRoom( + ctx context.Context, + request *api.QueryJoinedHostsInRoomRequest, + response *api.QueryJoinedHostsInRoomResponse, +) (err error) { + response.JoinedHosts, err = f.db.GetJoinedHosts(ctx, request.RoomID) + return +} + +// QueryJoinedHostServerNamesInRoom implements api.FederationSenderInternalAPI +func (f *FederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom( + ctx context.Context, + request *api.QueryJoinedHostServerNamesInRoomRequest, + response *api.QueryJoinedHostServerNamesInRoomResponse, +) (err error) { + joinedHosts, err := f.db.GetJoinedHosts(ctx, request.RoomID) + if err != nil { + return + } + + response.ServerNames = make([]gomatrixserverlib.ServerName, 0, len(joinedHosts)) + for _, host := range joinedHosts { + response.ServerNames = append(response.ServerNames, host.ServerName) + } + + // TODO: remove duplicates? + + return +} diff --git a/federationsender/producers/roomserver.go b/federationsender/producers/roomserver.go new file mode 100644 index 000000000..48aeed8cc --- /dev/null +++ b/federationsender/producers/roomserver.go @@ -0,0 +1,102 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package producers + +import ( + "context" + + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" +) + +// RoomserverProducer produces events for the roomserver to consume. +type RoomserverProducer struct { + InputAPI api.RoomserverInternalAPI + serverName gomatrixserverlib.ServerName +} + +// NewRoomserverProducer creates a new RoomserverProducer +func NewRoomserverProducer( + rsAPI api.RoomserverInternalAPI, serverName gomatrixserverlib.ServerName, +) *RoomserverProducer { + return &RoomserverProducer{ + InputAPI: rsAPI, + serverName: serverName, + } +} + +// SendInviteResponse drops an invite response back into the roomserver so that users +// already in the room will be notified of the new invite. The invite response is signed +// by the remote side. +func (c *RoomserverProducer) SendInviteResponse( + ctx context.Context, res gomatrixserverlib.RespInviteV2, roomVersion gomatrixserverlib.RoomVersion, +) (string, error) { + ev := res.Event.Headered(roomVersion) + ire := api.InputRoomEvent{ + Kind: api.KindNew, + Event: ev, + AuthEventIDs: ev.AuthEventIDs(), + SendAsServer: string(c.serverName), + TransactionID: nil, + } + return c.SendInputRoomEvents(ctx, []api.InputRoomEvent{ire}) +} + +// SendEventWithState writes an event with KindNew to the roomserver input log +// with the state at the event as KindOutlier before it. +func (c *RoomserverProducer) SendEventWithState( + ctx context.Context, state gomatrixserverlib.RespState, event gomatrixserverlib.HeaderedEvent, +) error { + outliers, err := state.Events() + if err != nil { + return err + } + + var ires []api.InputRoomEvent + for _, outlier := range outliers { + ires = append(ires, api.InputRoomEvent{ + Kind: api.KindOutlier, + Event: outlier.Headered(event.RoomVersion), + AuthEventIDs: outlier.AuthEventIDs(), + }) + } + + stateEventIDs := make([]string, len(state.StateEvents)) + for i := range state.StateEvents { + stateEventIDs[i] = state.StateEvents[i].EventID() + } + + ires = append(ires, api.InputRoomEvent{ + Kind: api.KindNew, + Event: event, + AuthEventIDs: event.AuthEventIDs(), + HasState: true, + StateEventIDs: stateEventIDs, + }) + + _, err = c.SendInputRoomEvents(ctx, ires) + return err +} + +// SendInputRoomEvents writes the given input room events to the roomserver input API. +func (c *RoomserverProducer) SendInputRoomEvents( + ctx context.Context, ires []api.InputRoomEvent, +) (eventID string, err error) { + request := api.InputRoomEventsRequest{InputRoomEvents: ires} + var response api.InputRoomEventsResponse + err = c.InputAPI.InputRoomEvents(ctx, &request, &response) + eventID = response.EventID + return +} diff --git a/federationsender/query/query.go b/federationsender/query/query.go deleted file mode 100644 index 8c35bb29e..000000000 --- a/federationsender/query/query.go +++ /dev/null @@ -1,88 +0,0 @@ -package query - -import ( - "context" - "encoding/json" - "net/http" - - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/federationsender/api" - "github.com/matrix-org/dendrite/federationsender/types" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" -) - -// FederationSenderQueryDatabase has the APIs needed to implement the query API. -type FederationSenderQueryDatabase interface { - GetJoinedHosts( - ctx context.Context, roomID string, - ) ([]types.JoinedHost, error) -} - -// FederationSenderQueryAPI is an implementation of api.FederationSenderQueryAPI -type FederationSenderQueryAPI struct { - DB FederationSenderQueryDatabase -} - -// QueryJoinedHostsInRoom implements api.FederationSenderQueryAPI -func (f *FederationSenderQueryAPI) QueryJoinedHostsInRoom( - ctx context.Context, - request *api.QueryJoinedHostsInRoomRequest, - response *api.QueryJoinedHostsInRoomResponse, -) (err error) { - response.JoinedHosts, err = f.DB.GetJoinedHosts(ctx, request.RoomID) - return -} - -// QueryJoinedHostServerNamesInRoom implements api.FederationSenderQueryAPI -func (f *FederationSenderQueryAPI) QueryJoinedHostServerNamesInRoom( - ctx context.Context, - request *api.QueryJoinedHostServerNamesInRoomRequest, - response *api.QueryJoinedHostServerNamesInRoomResponse, -) (err error) { - joinedHosts, err := f.DB.GetJoinedHosts(ctx, request.RoomID) - if err != nil { - return - } - - response.ServerNames = make([]gomatrixserverlib.ServerName, 0, len(joinedHosts)) - for _, host := range joinedHosts { - response.ServerNames = append(response.ServerNames, host.ServerName) - } - - // TODO: remove duplicates? - - return -} - -// SetupHTTP adds the FederationSenderQueryAPI handlers to the http.ServeMux. -func (f *FederationSenderQueryAPI) SetupHTTP(servMux *http.ServeMux) { - servMux.Handle( - api.FederationSenderQueryJoinedHostsInRoomPath, - common.MakeInternalAPI("QueryJoinedHostsInRoom", func(req *http.Request) util.JSONResponse { - var request api.QueryJoinedHostsInRoomRequest - var response api.QueryJoinedHostsInRoomResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := f.QueryJoinedHostsInRoom(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.FederationSenderQueryJoinedHostServerNamesInRoomPath, - common.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse { - var request api.QueryJoinedHostServerNamesInRoomRequest - var response api.QueryJoinedHostServerNamesInRoomResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := f.QueryJoinedHostServerNamesInRoom(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) -} diff --git a/federationsender/queue/destinationqueue.go b/federationsender/queue/destinationqueue.go index 7d4dc850b..89526fcfd 100644 --- a/federationsender/queue/destinationqueue.go +++ b/federationsender/queue/destinationqueue.go @@ -21,6 +21,7 @@ import ( "sync" "time" + "github.com/matrix-org/dendrite/federationsender/producers" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" log "github.com/sirupsen/logrus" @@ -32,6 +33,7 @@ import ( // ensures that only one request is in flight to a given destination // at a time. type destinationQueue struct { + rsProducer *producers.RoomserverProducer client *gomatrixserverlib.FederationClient origin gomatrixserverlib.ServerName destination gomatrixserverlib.ServerName @@ -165,18 +167,38 @@ func (oq *destinationQueue) nextInvites() bool { } for _, inviteReq := range oq.pendingInvites { - ev := inviteReq.Event() + ev, roomVersion := inviteReq.Event(), inviteReq.RoomVersion() - if _, err := oq.client.SendInviteV2( + log.WithFields(log.Fields{ + "event_id": ev.EventID(), + "room_version": roomVersion, + "destination": oq.destination, + }).Info("sending invite") + + inviteRes, err := oq.client.SendInviteV2( context.TODO(), oq.destination, *inviteReq, - ); err != nil { + ) + if err != nil { log.WithFields(log.Fields{ "event_id": ev.EventID(), "state_key": ev.StateKey(), "destination": oq.destination, }).WithError(err).Error("failed to send invite") + continue + } + + if _, err = oq.rsProducer.SendInviteResponse( + context.TODO(), + inviteRes, + roomVersion, + ); err != nil { + log.WithFields(log.Fields{ + "event_id": ev.EventID(), + "state_key": ev.StateKey(), + "destination": oq.destination, + }).WithError(err).Error("failed to return signed invite to roomserver") } } diff --git a/federationsender/queue/queue.go b/federationsender/queue/queue.go index 88d47f120..33abc8fdd 100644 --- a/federationsender/queue/queue.go +++ b/federationsender/queue/queue.go @@ -18,6 +18,7 @@ import ( "fmt" "sync" + "github.com/matrix-org/dendrite/federationsender/producers" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" ) @@ -25,19 +26,25 @@ import ( // OutgoingQueues is a collection of queues for sending transactions to other // matrix servers type OutgoingQueues struct { - origin gomatrixserverlib.ServerName - client *gomatrixserverlib.FederationClient + rsProducer *producers.RoomserverProducer + origin gomatrixserverlib.ServerName + client *gomatrixserverlib.FederationClient // The queuesMutex protects queues queuesMutex sync.Mutex queues map[gomatrixserverlib.ServerName]*destinationQueue } // NewOutgoingQueues makes a new OutgoingQueues -func NewOutgoingQueues(origin gomatrixserverlib.ServerName, client *gomatrixserverlib.FederationClient) *OutgoingQueues { +func NewOutgoingQueues( + origin gomatrixserverlib.ServerName, + client *gomatrixserverlib.FederationClient, + rsProducer *producers.RoomserverProducer, +) *OutgoingQueues { return &OutgoingQueues{ - origin: origin, - client: client, - queues: map[gomatrixserverlib.ServerName]*destinationQueue{}, + rsProducer: rsProducer, + origin: origin, + client: client, + queues: map[gomatrixserverlib.ServerName]*destinationQueue{}, } } @@ -67,6 +74,7 @@ func (oqs *OutgoingQueues) SendEvent( oq := oqs.queues[destination] if oq == nil { oq = &destinationQueue{ + rsProducer: oqs.rsProducer, origin: oqs.origin, destination: destination, client: oqs.client, @@ -111,6 +119,7 @@ func (oqs *OutgoingQueues) SendInvite( oq := oqs.queues[destination] if oq == nil { oq = &destinationQueue{ + rsProducer: oqs.rsProducer, origin: oqs.origin, destination: destination, client: oqs.client, @@ -151,6 +160,7 @@ func (oqs *OutgoingQueues) SendEDU( oq := oqs.queues[destination] if oq == nil { oq = &destinationQueue{ + rsProducer: oqs.rsProducer, origin: oqs.origin, destination: destination, client: oqs.client, diff --git a/federationsender/storage/postgres/storage.go b/federationsender/storage/postgres/storage.go index b909a189b..c3892ac10 100644 --- a/federationsender/storage/postgres/storage.go +++ b/federationsender/storage/postgres/storage.go @@ -33,10 +33,10 @@ type Database struct { } // NewDatabase opens a new database -func NewDatabase(dataSourceName string) (*Database, error) { +func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (*Database, error) { var result Database var err error - if result.db, err = sqlutil.Open("postgres", dataSourceName); err != nil { + if result.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/federationsender/storage/sqlite3/storage.go b/federationsender/storage/sqlite3/storage.go index 458d7d7e5..772744474 100644 --- a/federationsender/storage/sqlite3/storage.go +++ b/federationsender/storage/sqlite3/storage.go @@ -38,7 +38,7 @@ type Database struct { func NewDatabase(dataSourceName string) (*Database, error) { var result Database var err error - if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/federationsender/storage/storage.go b/federationsender/storage/storage.go index 2f018dff1..d481e58a2 100644 --- a/federationsender/storage/storage.go +++ b/federationsender/storage/storage.go @@ -19,22 +19,23 @@ package storage import ( "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/federationsender/storage/postgres" "github.com/matrix-org/dendrite/federationsender/storage/sqlite3" ) // NewDatabase opens a new database -func NewDatabase(dataSourceName string) (Database, error) { +func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.NewDatabase(dataSourceName) + return postgres.NewDatabase(dataSourceName, dbProperties) } switch uri.Scheme { case "file": return sqlite3.NewDatabase(dataSourceName) case "postgres": - return postgres.NewDatabase(dataSourceName) + return postgres.NewDatabase(dataSourceName, dbProperties) default: - return postgres.NewDatabase(dataSourceName) + return postgres.NewDatabase(dataSourceName, dbProperties) } } diff --git a/federationsender/storage/storage_wasm.go b/federationsender/storage/storage_wasm.go index f2c8ae1b4..44d4c8060 100644 --- a/federationsender/storage/storage_wasm.go +++ b/federationsender/storage/storage_wasm.go @@ -18,11 +18,15 @@ import ( "fmt" "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/federationsender/storage/sqlite3" ) // NewDatabase opens a new database -func NewDatabase(dataSourceName string) (Database, error) { +func NewDatabase( + dataSourceName string, + dbProperties common.DbProperties, // nolint:unparam +) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return nil, fmt.Errorf("Cannot use postgres implementation") diff --git a/federationsender/types/types.go b/federationsender/types/types.go index 05ba92f77..398d32677 100644 --- a/federationsender/types/types.go +++ b/federationsender/types/types.go @@ -28,6 +28,12 @@ type JoinedHost struct { ServerName gomatrixserverlib.ServerName } +type ServerNames []gomatrixserverlib.ServerName + +func (s ServerNames) Len() int { return len(s) } +func (s ServerNames) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s ServerNames) Less(i, j int) bool { return s[i] < s[j] } + // A EventIDMismatchError indicates that we have got out of sync with the // room server. type EventIDMismatchError struct { diff --git a/go.mod b/go.mod index 8d91902d1..55c1179c2 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,9 @@ module github.com/matrix-org/dendrite require ( + github.com/Shopify/sarama v1.26.1 github.com/gorilla/mux v1.7.3 github.com/hashicorp/golang-lru v0.5.4 - github.com/kr/pretty v0.2.0 // indirect github.com/lib/pq v1.2.0 github.com/libp2p/go-libp2p v0.6.0 github.com/libp2p/go-libp2p-circuit v0.1.4 @@ -17,26 +17,21 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200421090225-4ea81b29f5f7 - github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200430104311-8d41c4d924ec + github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 - github.com/mattn/go-sqlite3 v2.0.3+incompatible + github.com/mattn/go-sqlite3 v2.0.2+incompatible github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6 github.com/opentracing/opentracing-go v1.1.0 - github.com/pierrec/lz4 v2.5.0+incompatible // indirect github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.4.1 - github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect github.com/sirupsen/logrus v1.4.2 - github.com/tidwall/gjson v1.6.0 // indirect - github.com/tidwall/pretty v1.0.1 // indirect - github.com/uber/jaeger-client-go v2.22.1+incompatible - github.com/uber/jaeger-lib v2.2.0+incompatible - go.uber.org/atomic v1.6.0 + github.com/tidwall/gjson v1.6.0 + github.com/uber/jaeger-client-go v2.15.0+incompatible + github.com/uber/jaeger-lib v1.5.0 + go.uber.org/atomic v1.4.0 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d - golang.org/x/tools v0.0.0-20200402223321-bcf690261a44 // indirect - gopkg.in/Shopify/sarama.v1 v1.20.1 gopkg.in/h2non/bimg.v1 v1.0.18 gopkg.in/yaml.v2 v2.2.8 ) diff --git a/go.sum b/go.sum index e76f6d007..8c1d15ad7 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,12 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/zstd v1.4.4 h1:+IawcoXhCBylN7ccwdwf8LOH2jKq7NavGpEPanrlTzE= github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= +github.com/Shopify/sarama v1.26.1 h1:3jnfWKD7gVwbB1KSy/lE0szA9duPuSFLViK0o/d3DgA= +github.com/Shopify/sarama v1.26.1/go.mod h1:NbSGBSSndYaIhRcBtY9V0U7AyH+x71bG668AuWys/yU= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= @@ -61,19 +62,18 @@ github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUn github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= -github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= -github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k= github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk= github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -119,6 +119,8 @@ github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/U github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -140,7 +142,6 @@ github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67Fexh github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= -github.com/ipfs/go-datastore v0.3.1 h1:SS1t869a6cctoSYmZXUk8eL6AzVXgASmKIWFNQkQ1jU= github.com/ipfs/go-datastore v0.3.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-datastore v0.4.0/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= @@ -158,7 +159,6 @@ github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1 github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= -github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.2 h1:s19ZwJxH8rPWzypjcDpqPLIyV7BnbLqvpli3iZoqYK0= github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk= @@ -178,6 +178,8 @@ github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2/go.mod github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10= github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= +github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= +github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= @@ -188,9 +190,10 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= -github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA= +github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= @@ -364,10 +367,12 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200421090225-4ea81b29f5f7 h1:4vE84tE3r7BitCt2HQvT231JrhMjDfjDVDqVoiVPv0w= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200421090225-4ea81b29f5f7/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200430104311-8d41c4d924ec h1:9MvZSZzBKvCWqM5KXMGZ1PBDrSLcxs5zfc561UPgcYA= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200430104311-8d41c4d924ec/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= +github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= +github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5/go.mod h1:lePuOiXLNDott7NZfnQvJk0lAZ5HgvIuWGhel6J+RLA= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= @@ -380,8 +385,6 @@ github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-sqlite3 v2.0.2+incompatible h1:qzw9c2GNT8UFrgWNDhCTqRqYUSmu/Dav/9Z58LGpk7U= github.com/mattn/go-sqlite3 v2.0.2+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= -github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= @@ -475,8 +478,6 @@ github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg= github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4 v2.5.0+incompatible h1:MbdIZ43A//duwOjQqK3nP+up+65yraNFyX3Vp6Rwues= -github.com/pierrec/lz4 v2.5.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -505,10 +506,6 @@ github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLk github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= -github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= -github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= @@ -550,17 +547,12 @@ github.com/uber-go/atomic v1.3.0 h1:ylWoWcs+jXihgo3Us1Sdsatf2R6+OlBGm8fexR3oFG4= github.com/uber-go/atomic v1.3.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk= github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= -github.com/uber/jaeger-client-go v2.22.1+incompatible h1:NHcubEkVbahf9t3p75TOCR83gdUHXjRJvjoBh1yACsM= -github.com/uber/jaeger-client-go v2.22.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo= github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= -github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= -github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= -github.com/whyrusleeping/go-logging v0.0.1 h1:fwpzlmT0kRC/Fmd0MdmGgJG/CXIZ6gFq46FQZjprUcc= github.com/whyrusleeping/go-logging v0.0.1/go.mod h1:lDPYj54zutzG1XYfHAhcc7oNXEburHQBn+Iqd4yS4vE= github.com/whyrusleeping/mafmt v1.2.8 h1:TCghSl5kkwEE0j+sU/gudyhVMRlpBin8fMBBHg59EbA= github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvXwGvqIpk20FA= @@ -571,8 +563,9 @@ github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go. github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -580,9 +573,8 @@ go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.0 h1:vs7fgriifsPbGdK3bNuMWapNn3qnZhCRXc19NRdq010= go.uber.org/atomic v1.3.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= @@ -605,16 +597,13 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d h1:2+ZP7EfsZV7Vvmx3TIqSlSzATMkTAKqM14YGFPoSKjI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -629,6 +618,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -673,11 +663,6 @@ golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c h1:IGkKhmfzcztjm6gYkykvu/NiS8kaqbCWAEWWAyf8J5U= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200402223321-bcf690261a44 h1:bMm0eoDiGkM5VfIyKjxDvoflW5GLp7+VCo+60n8F+TE= -golang.org/x/tools v0.0.0-20200402223321-bcf690261a44/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -707,6 +692,16 @@ gopkg.in/h2non/bimg.v1 v1.0.18 h1:qn6/RpBHt+7WQqoBcK+aF2puc6nC78eZj5LexxoalT4= gopkg.in/h2non/bimg.v1 v1.0.18/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= gopkg.in/h2non/gock.v1 v1.0.14 h1:fTeu9fcUvSnLNacYvYI54h+1/XEteDyHvrVCZEEEYNM= gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= +gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw= +gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= +gopkg.in/jcmturner/goidentity.v3 v3.0.0 h1:1duIyWiTaYvVx3YX2CYtpJbUFd7/UuPYCfgXtQ3VTbI= +gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= +gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlIrg= +gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= +gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU= +gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= gopkg.in/macaroon.v2 v2.1.0 h1:HZcsjBCzq9t0eBPMKqTN/uSN6JOm78ZJ2INbqcBQOUI= gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= diff --git a/internal/sqlutil/trace.go b/internal/sqlutil/trace.go index 3d5fa7dc7..42ac4e582 100644 --- a/internal/sqlutil/trace.go +++ b/internal/sqlutil/trace.go @@ -21,9 +21,11 @@ import ( "fmt" "io" "os" + "regexp" "strings" "time" + "github.com/matrix-org/dendrite/common" "github.com/ngrok/sqlmw" "github.com/sirupsen/logrus" ) @@ -76,12 +78,27 @@ func (in *traceInterceptor) RowsNext(c context.Context, rows driver.Rows, dest [ // Open opens a database specified by its database driver name and a driver-specific data source name, // usually consisting of at least a database name and connection information. Includes tracing driver // if DENDRITE_TRACE_SQL=1 -func Open(driverName, dsn string) (*sql.DB, error) { +func Open(driverName, dsn string, dbProperties common.DbProperties) (*sql.DB, error) { if tracingEnabled { // install the wrapped driver driverName += "-trace" } - return sql.Open(driverName, dsn) + db, err := sql.Open(driverName, dsn) + if err != nil { + return nil, err + } + if driverName != common.SQLiteDriverName() && dbProperties != nil { + logrus.WithFields(logrus.Fields{ + "MaxOpenConns": dbProperties.MaxOpenConns(), + "MaxIdleConns": dbProperties.MaxIdleConns(), + "ConnMaxLifetime": dbProperties.ConnMaxLifetime(), + "dataSourceName": regexp.MustCompile(`://[^@]*@`).ReplaceAllLiteralString(dsn, "://"), + }).Debug("Setting DB connection limits") + db.SetMaxOpenConns(dbProperties.MaxOpenConns()) + db.SetMaxIdleConns(dbProperties.MaxIdleConns()) + db.SetConnMaxLifetime(dbProperties.ConnMaxLifetime()) + } + return db, nil } func init() { diff --git a/mediaapi/mediaapi.go b/mediaapi/mediaapi.go index f2e614c17..4a0f5d188 100644 --- a/mediaapi/mediaapi.go +++ b/mediaapi/mediaapi.go @@ -29,7 +29,7 @@ func SetupMediaAPIComponent( base *basecomponent.BaseDendrite, deviceDB devices.Database, ) { - mediaDB, err := storage.Open(string(base.Cfg.Database.MediaAPI)) + mediaDB, err := storage.Open(string(base.Cfg.Database.MediaAPI), base.Cfg.DbProperties()) if err != nil { logrus.WithError(err).Panicf("failed to connect to media db") } diff --git a/mediaapi/storage/postgres/storage.go b/mediaapi/storage/postgres/storage.go index 18126b151..4ddfc8fdf 100644 --- a/mediaapi/storage/postgres/storage.go +++ b/mediaapi/storage/postgres/storage.go @@ -21,6 +21,7 @@ import ( // Import the postgres database driver. _ "github.com/lib/pq" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -33,10 +34,10 @@ type Database struct { } // Open opens a postgres database. -func Open(dataSourceName string) (*Database, error) { +func Open(dataSourceName string, dbProperties common.DbProperties) (*Database, error) { var d Database var err error - if d.db, err = sqlutil.Open("postgres", dataSourceName); err != nil { + if d.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } if err = d.statements.prepare(d.db); err != nil { diff --git a/mediaapi/storage/sqlite3/storage.go b/mediaapi/storage/sqlite3/storage.go index abafecf20..8fa2e5375 100644 --- a/mediaapi/storage/sqlite3/storage.go +++ b/mediaapi/storage/sqlite3/storage.go @@ -37,7 +37,7 @@ type Database struct { func Open(dataSourceName string) (*Database, error) { var d Database var err error - if d.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if d.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil { return nil, err } if err = d.statements.prepare(d.db); err != nil { diff --git a/mediaapi/storage/storage.go b/mediaapi/storage/storage.go index c533477cd..6589c8304 100644 --- a/mediaapi/storage/storage.go +++ b/mediaapi/storage/storage.go @@ -19,22 +19,23 @@ package storage import ( "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/mediaapi/storage/postgres" "github.com/matrix-org/dendrite/mediaapi/storage/sqlite3" ) // Open opens a postgres database. -func Open(dataSourceName string) (Database, error) { +func Open(dataSourceName string, dbProperties common.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.Open(dataSourceName) + return postgres.Open(dataSourceName, dbProperties) } switch uri.Scheme { case "postgres": - return postgres.Open(dataSourceName) + return postgres.Open(dataSourceName, dbProperties) case "file": return sqlite3.Open(dataSourceName) default: - return postgres.Open(dataSourceName) + return postgres.Open(dataSourceName, dbProperties) } } diff --git a/mediaapi/storage/storage_wasm.go b/mediaapi/storage/storage_wasm.go index 92f0ad134..3c39e5d31 100644 --- a/mediaapi/storage/storage_wasm.go +++ b/mediaapi/storage/storage_wasm.go @@ -18,11 +18,15 @@ import ( "fmt" "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/mediaapi/storage/sqlite3" ) // Open opens a postgres database. -func Open(dataSourceName string) (Database, error) { +func Open( + dataSourceName string, + dbProperties common.DbProperties, // nolint:unparam +) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return nil, fmt.Errorf("Cannot use postgres implementation") diff --git a/publicroomsapi/consumers/roomserver.go b/publicroomsapi/consumers/roomserver.go index 2bbd92b72..efd093b7e 100644 --- a/publicroomsapi/consumers/roomserver.go +++ b/publicroomsapi/consumers/roomserver.go @@ -18,20 +18,20 @@ import ( "context" "encoding/json" + "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" - sarama "gopkg.in/Shopify/sarama.v1" ) // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { - roomServerConsumer *common.ContinualConsumer - db storage.Database - query api.RoomserverQueryAPI + rsAPI api.RoomserverInternalAPI + rsConsumer *common.ContinualConsumer + db storage.Database } // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. @@ -39,7 +39,7 @@ func NewOutputRoomEventConsumer( cfg *config.Dendrite, kafkaConsumer sarama.Consumer, store storage.Database, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, ) *OutputRoomEventConsumer { consumer := common.ContinualConsumer{ Topic: string(cfg.Kafka.Topics.OutputRoomEvent), @@ -47,9 +47,9 @@ func NewOutputRoomEventConsumer( PartitionStore: store, } s := &OutputRoomEventConsumer{ - roomServerConsumer: &consumer, - db: store, - query: queryAPI, + rsConsumer: &consumer, + db: store, + rsAPI: rsAPI, } consumer.ProcessMessage = s.onMessage @@ -58,7 +58,7 @@ func NewOutputRoomEventConsumer( // Start consuming from room servers func (s *OutputRoomEventConsumer) Start() error { - return s.roomServerConsumer.Start() + return s.rsConsumer.Start() } // onMessage is called when the sync server receives a new event from the room server output log. @@ -87,14 +87,14 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { addQueryReq := api.QueryEventsByIDRequest{EventIDs: output.NewRoomEvent.AddsStateEventIDs} var addQueryRes api.QueryEventsByIDResponse - if err := s.query.QueryEventsByID(context.TODO(), &addQueryReq, &addQueryRes); err != nil { + if err := s.rsAPI.QueryEventsByID(context.TODO(), &addQueryReq, &addQueryRes); err != nil { log.Warn(err) return err } remQueryReq := api.QueryEventsByIDRequest{EventIDs: output.NewRoomEvent.RemovesStateEventIDs} var remQueryRes api.QueryEventsByIDResponse - if err := s.query.QueryEventsByID(context.TODO(), &remQueryReq, &remQueryRes); err != nil { + if err := s.rsAPI.QueryEventsByID(context.TODO(), &remQueryReq, &remQueryRes); err != nil { log.Warn(err) return err } diff --git a/publicroomsapi/directory/directory.go b/publicroomsapi/directory/directory.go index 837018e64..fe7a67932 100644 --- a/publicroomsapi/directory/directory.go +++ b/publicroomsapi/directory/directory.go @@ -59,7 +59,7 @@ func GetVisibility( // SetVisibility implements PUT /directory/list/room/{roomID} // TODO: Allow admin users to edit the room visibility func SetVisibility( - req *http.Request, publicRoomsDatabase storage.Database, queryAPI api.RoomserverQueryAPI, dev *authtypes.Device, + req *http.Request, publicRoomsDatabase storage.Database, rsAPI api.RoomserverInternalAPI, dev *authtypes.Device, roomID string, ) util.JSONResponse { queryMembershipReq := api.QueryMembershipForUserRequest{ @@ -67,7 +67,7 @@ func SetVisibility( UserID: dev.UserID, } var queryMembershipRes api.QueryMembershipForUserResponse - err := queryAPI.QueryMembershipForUser(req.Context(), &queryMembershipReq, &queryMembershipRes) + err := rsAPI.QueryMembershipForUser(req.Context(), &queryMembershipReq, &queryMembershipRes) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("could not query membership for user") return jsonerror.InternalServerError() @@ -87,7 +87,7 @@ func SetVisibility( }}, } var queryEventsRes api.QueryLatestEventsAndStateResponse - err = queryAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes) + err = rsAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes) if err != nil || len(queryEventsRes.StateEvents) == 0 { util.GetLogger(req.Context()).WithError(err).Error("could not query events from room") return jsonerror.InternalServerError() diff --git a/publicroomsapi/publicroomsapi.go b/publicroomsapi/publicroomsapi.go index 6efb54bd9..6a4e65674 100644 --- a/publicroomsapi/publicroomsapi.go +++ b/publicroomsapi/publicroomsapi.go @@ -32,16 +32,16 @@ func SetupPublicRoomsAPIComponent( base *basecomponent.BaseDendrite, deviceDB devices.Database, publicRoomsDB storage.Database, - rsQueryAPI roomserverAPI.RoomserverQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider, ) { rsConsumer := consumers.NewOutputRoomEventConsumer( - base.Cfg, base.KafkaConsumer, publicRoomsDB, rsQueryAPI, + base.Cfg, base.KafkaConsumer, publicRoomsDB, rsAPI, ) if err := rsConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start public rooms server consumer") } - routing.Setup(base.APIMux, deviceDB, publicRoomsDB, rsQueryAPI, fedClient, extRoomsProvider) + routing.Setup(base.APIMux, deviceDB, publicRoomsDB, rsAPI, fedClient, extRoomsProvider) } diff --git a/publicroomsapi/routing/routing.go b/publicroomsapi/routing/routing.go index da5ea90d6..09a8eff76 100644 --- a/publicroomsapi/routing/routing.go +++ b/publicroomsapi/routing/routing.go @@ -39,7 +39,7 @@ const pathPrefixR0 = "/_matrix/client/r0" // applied: // nolint: gocyclo func Setup( - apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database, queryAPI api.RoomserverQueryAPI, + apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database, rsAPI api.RoomserverInternalAPI, fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider, ) { r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() @@ -66,7 +66,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return directory.SetVisibility(req, publicRoomsDB, queryAPI, device, vars["roomID"]) + return directory.SetVisibility(req, publicRoomsDB, rsAPI, device, vars["roomID"]) }), ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/publicRooms", diff --git a/publicroomsapi/storage/postgres/storage.go b/publicroomsapi/storage/postgres/storage.go index 8c4660cca..6242c9d54 100644 --- a/publicroomsapi/storage/postgres/storage.go +++ b/publicroomsapi/storage/postgres/storage.go @@ -36,10 +36,10 @@ type PublicRoomsServerDatabase struct { type attributeValue interface{} // NewPublicRoomsServerDatabase creates a new public rooms server database. -func NewPublicRoomsServerDatabase(dataSourceName string) (*PublicRoomsServerDatabase, error) { +func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties common.DbProperties) (*PublicRoomsServerDatabase, error) { var db *sql.DB var err error - if db, err = sqlutil.Open("postgres", dataSourceName); err != nil { + if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } storage := PublicRoomsServerDatabase{ diff --git a/publicroomsapi/storage/sqlite3/storage.go b/publicroomsapi/storage/sqlite3/storage.go index 121601628..efe35bdde 100644 --- a/publicroomsapi/storage/sqlite3/storage.go +++ b/publicroomsapi/storage/sqlite3/storage.go @@ -41,7 +41,7 @@ type attributeValue interface{} func NewPublicRoomsServerDatabase(dataSourceName string) (*PublicRoomsServerDatabase, error) { var db *sql.DB var err error - if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil { return nil, err } storage := PublicRoomsServerDatabase{ diff --git a/publicroomsapi/storage/storage.go b/publicroomsapi/storage/storage.go index e674514aa..7dcfe5633 100644 --- a/publicroomsapi/storage/storage.go +++ b/publicroomsapi/storage/storage.go @@ -19,6 +19,7 @@ package storage import ( "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/publicroomsapi/storage/postgres" "github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3" ) @@ -27,17 +28,17 @@ const schemePostgres = "postgres" const schemeFile = "file" // NewPublicRoomsServerDatabase opens a database connection. -func NewPublicRoomsServerDatabase(dataSourceName string) (Database, error) { +func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties common.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.NewPublicRoomsServerDatabase(dataSourceName) + return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties) } switch uri.Scheme { case schemePostgres: - return postgres.NewPublicRoomsServerDatabase(dataSourceName) + return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties) case schemeFile: return sqlite3.NewPublicRoomsServerDatabase(dataSourceName) default: - return postgres.NewPublicRoomsServerDatabase(dataSourceName) + return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties) } } diff --git a/roomserver/alias/alias_test.go b/roomserver/alias/alias_test.go deleted file mode 100644 index 0aefa19d9..000000000 --- a/roomserver/alias/alias_test.go +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright 2019 Serra Allgood -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package alias - -import ( - "context" - "fmt" - "strings" - "testing" - - appserviceAPI "github.com/matrix-org/dendrite/appservice/api" - roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/gomatrixserverlib" -) - -type MockRoomserverAliasAPIDatabase struct { - mode string - attempts int -} - -// These methods can be essentially noop -func (db MockRoomserverAliasAPIDatabase) SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error { - return nil -} - -func (db MockRoomserverAliasAPIDatabase) GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error) { - aliases := make([]string, 0) - return aliases, nil -} - -func (db MockRoomserverAliasAPIDatabase) RemoveRoomAlias(ctx context.Context, alias string) error { - return nil -} - -func (db *MockRoomserverAliasAPIDatabase) GetCreatorIDForAlias( - ctx context.Context, alias string, -) (string, error) { - return "", nil -} - -func (db *MockRoomserverAliasAPIDatabase) GetRoomVersionForRoom( - ctx context.Context, roomID string, -) (gomatrixserverlib.RoomVersion, error) { - return gomatrixserverlib.RoomVersionV1, nil -} - -// This method needs to change depending on test case -func (db *MockRoomserverAliasAPIDatabase) GetRoomIDForAlias( - ctx context.Context, - alias string, -) (string, error) { - switch db.mode { - case "empty": - return "", nil - case "error": - return "", fmt.Errorf("found an error from GetRoomIDForAlias") - case "found": - return "123", nil - case "emptyFound": - switch db.attempts { - case 0: - db.attempts = 1 - return "", nil - case 1: - db.attempts = 0 - return "123", nil - default: - return "", nil - } - default: - return "", fmt.Errorf("unknown option used") - } -} - -type MockAppServiceQueryAPI struct { - mode string -} - -// This method can be noop -func (q MockAppServiceQueryAPI) UserIDExists( - ctx context.Context, - req *appserviceAPI.UserIDExistsRequest, - resp *appserviceAPI.UserIDExistsResponse, -) error { - return nil -} - -func (q MockAppServiceQueryAPI) RoomAliasExists( - ctx context.Context, - req *appserviceAPI.RoomAliasExistsRequest, - resp *appserviceAPI.RoomAliasExistsResponse, -) error { - switch q.mode { - case "error": - return fmt.Errorf("found an error from RoomAliasExists") - case "found": - resp.AliasExists = true - return nil - case "empty": - resp.AliasExists = false - return nil - default: - return fmt.Errorf("Unknown option used") - } -} - -func TestGetRoomIDForAlias(t *testing.T) { - type arguments struct { - ctx context.Context - request *roomserverAPI.GetRoomIDForAliasRequest - response *roomserverAPI.GetRoomIDForAliasResponse - } - args := arguments{ - context.Background(), - &roomserverAPI.GetRoomIDForAliasRequest{}, - &roomserverAPI.GetRoomIDForAliasResponse{}, - } - type testCase struct { - name string - dbMode string - queryMode string - wantError bool - errorMsg string - want string - } - tt := []testCase{ - { - "found local alias", - "found", - "error", - false, - "", - "123", - }, - { - "found appservice alias", - "emptyFound", - "found", - false, - "", - "123", - }, - { - "error returned from DB", - "error", - "", - true, - "GetRoomIDForAlias", - "", - }, - { - "error returned from appserviceAPI", - "empty", - "error", - true, - "RoomAliasExists", - "", - }, - { - "no errors but no alias", - "empty", - "empty", - false, - "", - "", - }, - } - - setup := func(dbMode, queryMode string) *RoomserverAliasAPI { - mockAliasAPIDB := &MockRoomserverAliasAPIDatabase{dbMode, 0} - mockAppServiceQueryAPI := MockAppServiceQueryAPI{queryMode} - - return &RoomserverAliasAPI{ - DB: mockAliasAPIDB, - AppserviceAPI: mockAppServiceQueryAPI, - } - } - - for _, tc := range tt { - t.Run(tc.name, func(t *testing.T) { - aliasAPI := setup(tc.dbMode, tc.queryMode) - - err := aliasAPI.GetRoomIDForAlias(args.ctx, args.request, args.response) - if tc.wantError { - if err == nil { - t.Fatalf("Got no error; wanted error from %s", tc.errorMsg) - } else if !strings.Contains(err.Error(), tc.errorMsg) { - t.Fatalf("Got %s; wanted error from %s", err, tc.errorMsg) - } - } else if err != nil { - t.Fatalf("Got %s; wanted no error", err) - } else if args.response.RoomID != tc.want { - t.Errorf("Got '%s'; wanted '%s'", args.response.RoomID, tc.want) - } - }) - } -} diff --git a/roomserver/api/alias.go b/roomserver/api/alias.go index ad375a830..488e99ab8 100644 --- a/roomserver/api/alias.go +++ b/roomserver/api/alias.go @@ -16,8 +16,6 @@ package api import ( "context" - "errors" - "net/http" commonHTTP "github.com/matrix-org/dendrite/common/http" opentracing "github.com/opentracing/opentracing-go" @@ -86,44 +84,6 @@ type RemoveRoomAliasRequest struct { // RemoveRoomAliasResponse is a response to RemoveRoomAlias type RemoveRoomAliasResponse struct{} -// RoomserverAliasAPI is used to save, lookup or remove a room alias -type RoomserverAliasAPI interface { - // Set a room alias - SetRoomAlias( - ctx context.Context, - req *SetRoomAliasRequest, - response *SetRoomAliasResponse, - ) error - - // Get the room ID for an alias - GetRoomIDForAlias( - ctx context.Context, - req *GetRoomIDForAliasRequest, - response *GetRoomIDForAliasResponse, - ) error - - // Get all known aliases for a room ID - GetAliasesForRoomID( - ctx context.Context, - req *GetAliasesForRoomIDRequest, - response *GetAliasesForRoomIDResponse, - ) error - - // Get the user ID of the creator of an alias - GetCreatorIDForAlias( - ctx context.Context, - req *GetCreatorIDForAliasRequest, - response *GetCreatorIDForAliasResponse, - ) error - - // Remove a room alias - RemoveRoomAlias( - ctx context.Context, - req *RemoveRoomAliasRequest, - response *RemoveRoomAliasResponse, - ) error -} - // RoomserverSetRoomAliasPath is the HTTP path for the SetRoomAlias API. const RoomserverSetRoomAliasPath = "/api/roomserver/setRoomAlias" @@ -139,22 +99,8 @@ const RoomserverGetCreatorIDForAliasPath = "/api/roomserver/GetCreatorIDForAlias // RoomserverRemoveRoomAliasPath is the HTTP path for the RemoveRoomAlias API. const RoomserverRemoveRoomAliasPath = "/api/roomserver/removeRoomAlias" -// NewRoomserverAliasAPIHTTP creates a RoomserverAliasAPI implemented by talking to a HTTP POST API. -// If httpClient is nil an error is returned -func NewRoomserverAliasAPIHTTP(roomserverURL string, httpClient *http.Client) (RoomserverAliasAPI, error) { - if httpClient == nil { - return nil, errors.New("NewRoomserverAliasAPIHTTP: httpClient is ") - } - return &httpRoomserverAliasAPI{roomserverURL, httpClient}, nil -} - -type httpRoomserverAliasAPI struct { - roomserverURL string - httpClient *http.Client -} - // SetRoomAlias implements RoomserverAliasAPI -func (h *httpRoomserverAliasAPI) SetRoomAlias( +func (h *httpRoomserverInternalAPI) SetRoomAlias( ctx context.Context, request *SetRoomAliasRequest, response *SetRoomAliasResponse, @@ -167,7 +113,7 @@ func (h *httpRoomserverAliasAPI) SetRoomAlias( } // GetRoomIDForAlias implements RoomserverAliasAPI -func (h *httpRoomserverAliasAPI) GetRoomIDForAlias( +func (h *httpRoomserverInternalAPI) GetRoomIDForAlias( ctx context.Context, request *GetRoomIDForAliasRequest, response *GetRoomIDForAliasResponse, @@ -180,7 +126,7 @@ func (h *httpRoomserverAliasAPI) GetRoomIDForAlias( } // GetAliasesForRoomID implements RoomserverAliasAPI -func (h *httpRoomserverAliasAPI) GetAliasesForRoomID( +func (h *httpRoomserverInternalAPI) GetAliasesForRoomID( ctx context.Context, request *GetAliasesForRoomIDRequest, response *GetAliasesForRoomIDResponse, @@ -193,7 +139,7 @@ func (h *httpRoomserverAliasAPI) GetAliasesForRoomID( } // GetCreatorIDForAlias implements RoomserverAliasAPI -func (h *httpRoomserverAliasAPI) GetCreatorIDForAlias( +func (h *httpRoomserverInternalAPI) GetCreatorIDForAlias( ctx context.Context, request *GetCreatorIDForAliasRequest, response *GetCreatorIDForAliasResponse, @@ -206,7 +152,7 @@ func (h *httpRoomserverAliasAPI) GetCreatorIDForAlias( } // RemoveRoomAlias implements RoomserverAliasAPI -func (h *httpRoomserverAliasAPI) RemoveRoomAlias( +func (h *httpRoomserverInternalAPI) RemoveRoomAlias( ctx context.Context, request *RemoveRoomAliasRequest, response *RemoveRoomAliasResponse, diff --git a/roomserver/api/api.go b/roomserver/api/api.go new file mode 100644 index 000000000..ae4beab21 --- /dev/null +++ b/roomserver/api/api.go @@ -0,0 +1,147 @@ +package api + +import ( + "context" + + fsAPI "github.com/matrix-org/dendrite/federationsender/api" +) + +// RoomserverInputAPI is used to write events to the room server. +type RoomserverInternalAPI interface { + // needed to avoid chicken and egg scenario when setting up the + // interdependencies between the roomserver and other input APIs + SetFederationSenderAPI(fsAPI fsAPI.FederationSenderInternalAPI) + + InputRoomEvents( + ctx context.Context, + request *InputRoomEventsRequest, + response *InputRoomEventsResponse, + ) error + + PerformJoin( + ctx context.Context, + req *PerformJoinRequest, + res *PerformJoinResponse, + ) error + + // Query the latest events and state for a room from the room server. + QueryLatestEventsAndState( + ctx context.Context, + request *QueryLatestEventsAndStateRequest, + response *QueryLatestEventsAndStateResponse, + ) error + + // Query the state after a list of events in a room from the room server. + QueryStateAfterEvents( + ctx context.Context, + request *QueryStateAfterEventsRequest, + response *QueryStateAfterEventsResponse, + ) error + + // Query a list of events by event ID. + QueryEventsByID( + ctx context.Context, + request *QueryEventsByIDRequest, + response *QueryEventsByIDResponse, + ) error + + // Query the membership event for an user for a room. + QueryMembershipForUser( + ctx context.Context, + request *QueryMembershipForUserRequest, + response *QueryMembershipForUserResponse, + ) error + + // Query a list of membership events for a room + QueryMembershipsForRoom( + ctx context.Context, + request *QueryMembershipsForRoomRequest, + response *QueryMembershipsForRoomResponse, + ) error + + // Query a list of invite event senders for a user in a room. + QueryInvitesForUser( + ctx context.Context, + request *QueryInvitesForUserRequest, + response *QueryInvitesForUserResponse, + ) error + + // Query whether a server is allowed to see an event + QueryServerAllowedToSeeEvent( + ctx context.Context, + request *QueryServerAllowedToSeeEventRequest, + response *QueryServerAllowedToSeeEventResponse, + ) error + + // Query missing events for a room from roomserver + QueryMissingEvents( + ctx context.Context, + request *QueryMissingEventsRequest, + response *QueryMissingEventsResponse, + ) error + + // Query to get state and auth chain for a (potentially hypothetical) event. + // Takes lists of PrevEventIDs and AuthEventsIDs and uses them to calculate + // the state and auth chain to return. + QueryStateAndAuthChain( + ctx context.Context, + request *QueryStateAndAuthChainRequest, + response *QueryStateAndAuthChainResponse, + ) error + + // Query a given amount (or less) of events prior to a given set of events. + QueryBackfill( + ctx context.Context, + request *QueryBackfillRequest, + response *QueryBackfillResponse, + ) error + + // Asks for the default room version as preferred by the server. + QueryRoomVersionCapabilities( + ctx context.Context, + request *QueryRoomVersionCapabilitiesRequest, + response *QueryRoomVersionCapabilitiesResponse, + ) error + + // Asks for the room version for a given room. + QueryRoomVersionForRoom( + ctx context.Context, + request *QueryRoomVersionForRoomRequest, + response *QueryRoomVersionForRoomResponse, + ) error + + // Set a room alias + SetRoomAlias( + ctx context.Context, + req *SetRoomAliasRequest, + response *SetRoomAliasResponse, + ) error + + // Get the room ID for an alias + GetRoomIDForAlias( + ctx context.Context, + req *GetRoomIDForAliasRequest, + response *GetRoomIDForAliasResponse, + ) error + + // Get all known aliases for a room ID + GetAliasesForRoomID( + ctx context.Context, + req *GetAliasesForRoomIDRequest, + response *GetAliasesForRoomIDResponse, + ) error + + // Get the user ID of the creator of an alias + GetCreatorIDForAlias( + ctx context.Context, + req *GetCreatorIDForAliasRequest, + response *GetCreatorIDForAliasResponse, + ) error + + // Remove a room alias + RemoveRoomAlias( + ctx context.Context, + req *RemoveRoomAliasRequest, + response *RemoveRoomAliasResponse, + ) error +} diff --git a/roomserver/api/http.go b/roomserver/api/http.go new file mode 100644 index 000000000..d643526bd --- /dev/null +++ b/roomserver/api/http.go @@ -0,0 +1,41 @@ +package api + +import ( + "errors" + "net/http" + + "github.com/matrix-org/dendrite/common/caching" + fsInputAPI "github.com/matrix-org/dendrite/federationsender/api" +) + +type httpRoomserverInternalAPI struct { + roomserverURL string + httpClient *http.Client + fsAPI fsInputAPI.FederationSenderInternalAPI + immutableCache caching.ImmutableCache +} + +// NewRoomserverInputAPIHTTP creates a RoomserverInputAPI implemented by talking to a HTTP POST API. +// If httpClient is nil an error is returned +func NewRoomserverInternalAPIHTTP( + roomserverURL string, + httpClient *http.Client, + //fsInputAPI fsAPI.FederationSenderInternalAPI, + immutableCache caching.ImmutableCache, +) (RoomserverInternalAPI, error) { + if httpClient == nil { + return nil, errors.New("NewRoomserverInternalAPIHTTP: httpClient is ") + } + return &httpRoomserverInternalAPI{ + roomserverURL: roomserverURL, + httpClient: httpClient, + immutableCache: immutableCache, + }, nil +} + +// SetFederationSenderInputAPI passes in a federation sender input API reference +// so that we can avoid the chicken-and-egg problem of both the roomserver input API +// and the federation sender input API being interdependent. +func (h *httpRoomserverInternalAPI) SetFederationSenderAPI(fsAPI fsInputAPI.FederationSenderInternalAPI) { + h.fsAPI = fsAPI +} diff --git a/roomserver/api/input.go b/roomserver/api/input.go index 87e3983e3..8e8fdae4e 100644 --- a/roomserver/api/input.go +++ b/roomserver/api/input.go @@ -17,8 +17,6 @@ package api import ( "context" - "errors" - "net/http" commonHTTP "github.com/matrix-org/dendrite/common/http" "github.com/matrix-org/gomatrixserverlib" @@ -89,6 +87,8 @@ type InputInviteEvent struct { RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` Event gomatrixserverlib.HeaderedEvent `json:"event"` InviteRoomState []gomatrixserverlib.InviteV2StrippedState `json:"invite_room_state"` + SendAsServer string `json:"send_as_server"` + TransactionID *TransactionID `json:"transaction_id"` } // InputRoomEventsRequest is a request to InputRoomEvents @@ -102,34 +102,11 @@ type InputRoomEventsResponse struct { EventID string `json:"event_id"` } -// RoomserverInputAPI is used to write events to the room server. -type RoomserverInputAPI interface { - InputRoomEvents( - ctx context.Context, - request *InputRoomEventsRequest, - response *InputRoomEventsResponse, - ) error -} - // RoomserverInputRoomEventsPath is the HTTP path for the InputRoomEvents API. const RoomserverInputRoomEventsPath = "/api/roomserver/inputRoomEvents" -// NewRoomserverInputAPIHTTP creates a RoomserverInputAPI implemented by talking to a HTTP POST API. -// If httpClient is nil an error is returned -func NewRoomserverInputAPIHTTP(roomserverURL string, httpClient *http.Client) (RoomserverInputAPI, error) { - if httpClient == nil { - return nil, errors.New("NewRoomserverInputAPIHTTP: httpClient is ") - } - return &httpRoomserverInputAPI{roomserverURL, httpClient}, nil -} - -type httpRoomserverInputAPI struct { - roomserverURL string - httpClient *http.Client -} - // InputRoomEvents implements RoomserverInputAPI -func (h *httpRoomserverInputAPI) InputRoomEvents( +func (h *httpRoomserverInternalAPI) InputRoomEvents( ctx context.Context, request *InputRoomEventsRequest, response *InputRoomEventsResponse, diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go new file mode 100644 index 000000000..e60c078bc --- /dev/null +++ b/roomserver/api/perform.go @@ -0,0 +1,59 @@ +package api + +import ( + "context" + + commonHTTP "github.com/matrix-org/dendrite/common/http" + "github.com/matrix-org/gomatrixserverlib" + "github.com/opentracing/opentracing-go" +) + +const ( + // RoomserverPerformJoinPath is the HTTP path for the PerformJoin API. + RoomserverPerformJoinPath = "/api/roomserver/performJoin" + + // RoomserverPerformLeavePath is the HTTP path for the PerformLeave API. + RoomserverPerformLeavePath = "/api/roomserver/performLeave" +) + +type PerformJoinRequest struct { + RoomIDOrAlias string `json:"room_id_or_alias"` + UserID string `json:"user_id"` + Content map[string]interface{} `json:"content"` + ServerNames []gomatrixserverlib.ServerName `json:"server_names"` +} + +type PerformJoinResponse struct { +} + +func (h *httpRoomserverInternalAPI) PerformJoin( + ctx context.Context, + request *PerformJoinRequest, + response *PerformJoinResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformJoin") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverPerformJoinPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +type PerformLeaveRequest struct { + RoomID string `json:"room_id"` + UserID string `json:"user_id"` +} + +type PerformLeaveResponse struct { +} + +func (h *httpRoomserverInternalAPI) PerformLeave( + ctx context.Context, + request *PerformLeaveRequest, + response *PerformLeaveResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformLeave") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverPerformLeavePath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} diff --git a/roomserver/api/query.go b/roomserver/api/query.go index b272b1ebd..cb7cbb86c 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -18,10 +18,7 @@ package api import ( "context" - "errors" - "net/http" - "github.com/matrix-org/dendrite/common/caching" commonHTTP "github.com/matrix-org/dendrite/common/http" "github.com/matrix-org/gomatrixserverlib" opentracing "github.com/opentracing/opentracing-go" @@ -229,6 +226,8 @@ type QueryStateAndAuthChainResponse struct { // QueryBackfillRequest is a request to QueryBackfill. type QueryBackfillRequest struct { + // The room to backfill + RoomID string `json:"room_id"` // Events to start paginating from. EarliestEventsIDs []string `json:"earliest_event_ids"` // The maximum number of events to retrieve. @@ -243,21 +242,7 @@ type QueryBackfillResponse struct { Events []gomatrixserverlib.HeaderedEvent `json:"events"` } -// QueryServersInRoomAtEventRequest is a request to QueryServersInRoomAtEvent -type QueryServersInRoomAtEventRequest struct { - // ID of the room to retrieve member servers for. - RoomID string `json:"room_id"` - // ID of the event for which to retrieve member servers. - EventID string `json:"event_id"` -} - -// QueryServersInRoomAtEventResponse is a response to QueryServersInRoomAtEvent -type QueryServersInRoomAtEventResponse struct { - // Servers present in the room for these events. - Servers []gomatrixserverlib.ServerName `json:"servers"` -} - -// QueryRoomVersionCapabilities asks for the default room version +// QueryRoomVersionCapabilitiesRequest asks for the default room version type QueryRoomVersionCapabilitiesRequest struct{} // QueryRoomVersionCapabilitiesResponse is a response to QueryRoomVersionCapabilitiesRequest @@ -266,111 +251,16 @@ type QueryRoomVersionCapabilitiesResponse struct { AvailableRoomVersions map[gomatrixserverlib.RoomVersion]string `json:"available"` } -// QueryRoomVersionForRoom asks for the room version for a given room. +// QueryRoomVersionForRoomRequest asks for the room version for a given room. type QueryRoomVersionForRoomRequest struct { RoomID string `json:"room_id"` } -// QueryRoomVersionCapabilitiesResponse is a response to QueryServersInRoomAtEventResponse +// QueryRoomVersionForRoomResponse is a response to QueryRoomVersionForRoomRequest type QueryRoomVersionForRoomResponse struct { RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` } -// RoomserverQueryAPI is used to query information from the room server. -type RoomserverQueryAPI interface { - // Query the latest events and state for a room from the room server. - QueryLatestEventsAndState( - ctx context.Context, - request *QueryLatestEventsAndStateRequest, - response *QueryLatestEventsAndStateResponse, - ) error - - // Query the state after a list of events in a room from the room server. - QueryStateAfterEvents( - ctx context.Context, - request *QueryStateAfterEventsRequest, - response *QueryStateAfterEventsResponse, - ) error - - // Query a list of events by event ID. - QueryEventsByID( - ctx context.Context, - request *QueryEventsByIDRequest, - response *QueryEventsByIDResponse, - ) error - - // Query the membership event for an user for a room. - QueryMembershipForUser( - ctx context.Context, - request *QueryMembershipForUserRequest, - response *QueryMembershipForUserResponse, - ) error - - // Query a list of membership events for a room - QueryMembershipsForRoom( - ctx context.Context, - request *QueryMembershipsForRoomRequest, - response *QueryMembershipsForRoomResponse, - ) error - - // Query a list of invite event senders for a user in a room. - QueryInvitesForUser( - ctx context.Context, - request *QueryInvitesForUserRequest, - response *QueryInvitesForUserResponse, - ) error - - // Query whether a server is allowed to see an event - QueryServerAllowedToSeeEvent( - ctx context.Context, - request *QueryServerAllowedToSeeEventRequest, - response *QueryServerAllowedToSeeEventResponse, - ) error - - // Query missing events for a room from roomserver - QueryMissingEvents( - ctx context.Context, - request *QueryMissingEventsRequest, - response *QueryMissingEventsResponse, - ) error - - // Query to get state and auth chain for a (potentially hypothetical) event. - // Takes lists of PrevEventIDs and AuthEventsIDs and uses them to calculate - // the state and auth chain to return. - QueryStateAndAuthChain( - ctx context.Context, - request *QueryStateAndAuthChainRequest, - response *QueryStateAndAuthChainResponse, - ) error - - // Query a given amount (or less) of events prior to a given set of events. - QueryBackfill( - ctx context.Context, - request *QueryBackfillRequest, - response *QueryBackfillResponse, - ) error - - QueryServersInRoomAtEvent( - ctx context.Context, - request *QueryServersInRoomAtEventRequest, - response *QueryServersInRoomAtEventResponse, - ) error - - // Asks for the default room version as preferred by the server. - QueryRoomVersionCapabilities( - ctx context.Context, - request *QueryRoomVersionCapabilitiesRequest, - response *QueryRoomVersionCapabilitiesResponse, - ) error - - // Asks for the room version for a given room. - QueryRoomVersionForRoom( - ctx context.Context, - request *QueryRoomVersionForRoomRequest, - response *QueryRoomVersionForRoomResponse, - ) error -} - // RoomserverQueryLatestEventsAndStatePath is the HTTP path for the QueryLatestEventsAndState API. const RoomserverQueryLatestEventsAndStatePath = "/api/roomserver/queryLatestEventsAndState" @@ -401,32 +291,14 @@ const RoomserverQueryStateAndAuthChainPath = "/api/roomserver/queryStateAndAuthC // RoomserverQueryBackfillPath is the HTTP path for the QueryBackfillPath API const RoomserverQueryBackfillPath = "/api/roomserver/queryBackfill" -// RoomserverQueryServersInRoomAtEventPath is the HTTP path for the QueryServersInRoomAtEvent API -const RoomserverQueryServersInRoomAtEventPath = "/api/roomserver/queryServersInRoomAtEvents" - // RoomserverQueryRoomVersionCapabilitiesPath is the HTTP path for the QueryRoomVersionCapabilities API const RoomserverQueryRoomVersionCapabilitiesPath = "/api/roomserver/queryRoomVersionCapabilities" -// RoomserverQueryRoomVersionCapabilitiesPath is the HTTP path for the QueryRoomVersionCapabilities API +// RoomserverQueryRoomVersionForRoomPath is the HTTP path for the QueryRoomVersionForRoom API const RoomserverQueryRoomVersionForRoomPath = "/api/roomserver/queryRoomVersionForRoom" -// NewRoomserverQueryAPIHTTP creates a RoomserverQueryAPI implemented by talking to a HTTP POST API. -// If httpClient is nil an error is returned -func NewRoomserverQueryAPIHTTP(roomserverURL string, httpClient *http.Client, cache caching.ImmutableCache) (RoomserverQueryAPI, error) { - if httpClient == nil { - return nil, errors.New("NewRoomserverQueryAPIHTTP: httpClient is ") - } - return &httpRoomserverQueryAPI{roomserverURL, httpClient, cache}, nil -} - -type httpRoomserverQueryAPI struct { - roomserverURL string - httpClient *http.Client - immutableCache caching.ImmutableCache -} - // QueryLatestEventsAndState implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryLatestEventsAndState( +func (h *httpRoomserverInternalAPI) QueryLatestEventsAndState( ctx context.Context, request *QueryLatestEventsAndStateRequest, response *QueryLatestEventsAndStateResponse, @@ -439,7 +311,7 @@ func (h *httpRoomserverQueryAPI) QueryLatestEventsAndState( } // QueryStateAfterEvents implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryStateAfterEvents( +func (h *httpRoomserverInternalAPI) QueryStateAfterEvents( ctx context.Context, request *QueryStateAfterEventsRequest, response *QueryStateAfterEventsResponse, @@ -452,7 +324,7 @@ func (h *httpRoomserverQueryAPI) QueryStateAfterEvents( } // QueryEventsByID implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryEventsByID( +func (h *httpRoomserverInternalAPI) QueryEventsByID( ctx context.Context, request *QueryEventsByIDRequest, response *QueryEventsByIDResponse, @@ -465,7 +337,7 @@ func (h *httpRoomserverQueryAPI) QueryEventsByID( } // QueryMembershipForUser implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryMembershipForUser( +func (h *httpRoomserverInternalAPI) QueryMembershipForUser( ctx context.Context, request *QueryMembershipForUserRequest, response *QueryMembershipForUserResponse, @@ -478,7 +350,7 @@ func (h *httpRoomserverQueryAPI) QueryMembershipForUser( } // QueryMembershipsForRoom implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryMembershipsForRoom( +func (h *httpRoomserverInternalAPI) QueryMembershipsForRoom( ctx context.Context, request *QueryMembershipsForRoomRequest, response *QueryMembershipsForRoomResponse, @@ -491,7 +363,7 @@ func (h *httpRoomserverQueryAPI) QueryMembershipsForRoom( } // QueryInvitesForUser implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryInvitesForUser( +func (h *httpRoomserverInternalAPI) QueryInvitesForUser( ctx context.Context, request *QueryInvitesForUserRequest, response *QueryInvitesForUserResponse, @@ -504,7 +376,7 @@ func (h *httpRoomserverQueryAPI) QueryInvitesForUser( } // QueryServerAllowedToSeeEvent implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryServerAllowedToSeeEvent( +func (h *httpRoomserverInternalAPI) QueryServerAllowedToSeeEvent( ctx context.Context, request *QueryServerAllowedToSeeEventRequest, response *QueryServerAllowedToSeeEventResponse, @@ -517,7 +389,7 @@ func (h *httpRoomserverQueryAPI) QueryServerAllowedToSeeEvent( } // QueryMissingEvents implements RoomServerQueryAPI -func (h *httpRoomserverQueryAPI) QueryMissingEvents( +func (h *httpRoomserverInternalAPI) QueryMissingEvents( ctx context.Context, request *QueryMissingEventsRequest, response *QueryMissingEventsResponse, @@ -530,7 +402,7 @@ func (h *httpRoomserverQueryAPI) QueryMissingEvents( } // QueryStateAndAuthChain implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryStateAndAuthChain( +func (h *httpRoomserverInternalAPI) QueryStateAndAuthChain( ctx context.Context, request *QueryStateAndAuthChainRequest, response *QueryStateAndAuthChainResponse, @@ -543,7 +415,7 @@ func (h *httpRoomserverQueryAPI) QueryStateAndAuthChain( } // QueryBackfill implements RoomServerQueryAPI -func (h *httpRoomserverQueryAPI) QueryBackfill( +func (h *httpRoomserverInternalAPI) QueryBackfill( ctx context.Context, request *QueryBackfillRequest, response *QueryBackfillResponse, @@ -555,21 +427,8 @@ func (h *httpRoomserverQueryAPI) QueryBackfill( return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } -// QueryServersInRoomAtEvent implements RoomServerQueryAPI -func (h *httpRoomserverQueryAPI) QueryServersInRoomAtEvent( - ctx context.Context, - request *QueryServersInRoomAtEventRequest, - response *QueryServersInRoomAtEventResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryServersInRoomAtEvent") - defer span.Finish() - - apiURL := h.roomserverURL + RoomserverQueryServersInRoomAtEventPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - // QueryRoomVersionCapabilities implements RoomServerQueryAPI -func (h *httpRoomserverQueryAPI) QueryRoomVersionCapabilities( +func (h *httpRoomserverInternalAPI) QueryRoomVersionCapabilities( ctx context.Context, request *QueryRoomVersionCapabilitiesRequest, response *QueryRoomVersionCapabilitiesResponse, @@ -582,7 +441,7 @@ func (h *httpRoomserverQueryAPI) QueryRoomVersionCapabilities( } // QueryRoomVersionForRoom implements RoomServerQueryAPI -func (h *httpRoomserverQueryAPI) QueryRoomVersionForRoom( +func (h *httpRoomserverInternalAPI) QueryRoomVersionForRoom( ctx context.Context, request *QueryRoomVersionForRoomRequest, response *QueryRoomVersionForRoomResponse, diff --git a/roomserver/auth/auth.go b/roomserver/auth/auth.go index 615a94b3c..fdcf9f062 100644 --- a/roomserver/auth/auth.go +++ b/roomserver/auth/auth.go @@ -27,7 +27,7 @@ func IsServerAllowed( serverCurrentlyInRoom bool, authEvents []gomatrixserverlib.Event, ) bool { - historyVisibility := historyVisibilityForRoom(authEvents) + historyVisibility := HistoryVisibilityForRoom(authEvents) // 1. If the history_visibility was set to world_readable, allow. if historyVisibility == "world_readable" { @@ -52,7 +52,7 @@ func IsServerAllowed( return false } -func historyVisibilityForRoom(authEvents []gomatrixserverlib.Event) string { +func HistoryVisibilityForRoom(authEvents []gomatrixserverlib.Event) string { // https://matrix.org/docs/spec/client_server/r0.6.0#id87 // By default if no history_visibility is set, or if the value is not understood, the visibility is assumed to be shared. visibility := "shared" diff --git a/roomserver/alias/alias.go b/roomserver/internal/alias.go similarity index 50% rename from roomserver/alias/alias.go rename to roomserver/internal/alias.go index eb606e5cd..4139582b6 100644 --- a/roomserver/alias/alias.go +++ b/roomserver/internal/alias.go @@ -12,25 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -package alias +package internal import ( "context" "encoding/json" "errors" - "net/http" "time" - appserviceAPI "github.com/matrix-org/dendrite/appservice/api" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" - roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" ) -// RoomserverAliasAPIDatabase has the storage APIs needed to implement the alias API. -type RoomserverAliasAPIDatabase interface { +// RoomserverInternalAPIDatabase has the storage APIs needed to implement the alias API. +type RoomserverInternalAPIDatabase interface { // Save a given room alias with the room ID it refers to. // Returns an error if there was a problem talking to the database. SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error @@ -52,20 +47,11 @@ type RoomserverAliasAPIDatabase interface { ) (gomatrixserverlib.RoomVersion, error) } -// RoomserverAliasAPI is an implementation of alias.RoomserverAliasAPI -type RoomserverAliasAPI struct { - DB RoomserverAliasAPIDatabase - Cfg *config.Dendrite - InputAPI roomserverAPI.RoomserverInputAPI - QueryAPI roomserverAPI.RoomserverQueryAPI - AppserviceAPI appserviceAPI.AppServiceQueryAPI -} - -// SetRoomAlias implements alias.RoomserverAliasAPI -func (r *RoomserverAliasAPI) SetRoomAlias( +// SetRoomAlias implements alias.RoomserverInternalAPI +func (r *RoomserverInternalAPI) SetRoomAlias( ctx context.Context, - request *roomserverAPI.SetRoomAliasRequest, - response *roomserverAPI.SetRoomAliasResponse, + request *api.SetRoomAliasRequest, + response *api.SetRoomAliasResponse, ) error { // Check if the alias isn't already referring to a room roomID, err := r.DB.GetRoomIDForAlias(ctx, request.Alias) @@ -91,11 +77,11 @@ func (r *RoomserverAliasAPI) SetRoomAlias( return r.sendUpdatedAliasesEvent(context.TODO(), request.UserID, request.RoomID) } -// GetRoomIDForAlias implements alias.RoomserverAliasAPI -func (r *RoomserverAliasAPI) GetRoomIDForAlias( +// GetRoomIDForAlias implements alias.RoomserverInternalAPI +func (r *RoomserverInternalAPI) GetRoomIDForAlias( ctx context.Context, - request *roomserverAPI.GetRoomIDForAliasRequest, - response *roomserverAPI.GetRoomIDForAliasResponse, + request *api.GetRoomIDForAliasRequest, + response *api.GetRoomIDForAliasResponse, ) error { // Look up the room ID in the database roomID, err := r.DB.GetRoomIDForAlias(ctx, request.Alias) @@ -103,32 +89,38 @@ func (r *RoomserverAliasAPI) GetRoomIDForAlias( return err } - if roomID == "" { - // No room found locally, try our application services by making a call to - // the appservice component - aliasReq := appserviceAPI.RoomAliasExistsRequest{Alias: request.Alias} - var aliasResp appserviceAPI.RoomAliasExistsResponse - if err = r.AppserviceAPI.RoomAliasExists(ctx, &aliasReq, &aliasResp); err != nil { - return err - } + /* + TODO: Why is this here? It creates an unnecessary dependency + from the roomserver to the appservice component, which should be + altogether optional. - if aliasResp.AliasExists { - roomID, err = r.DB.GetRoomIDForAlias(ctx, request.Alias) - if err != nil { + if roomID == "" { + // No room found locally, try our application services by making a call to + // the appservice component + aliasReq := appserviceAPI.RoomAliasExistsRequest{Alias: request.Alias} + var aliasResp appserviceAPI.RoomAliasExistsResponse + if err = r.AppserviceAPI.RoomAliasExists(ctx, &aliasReq, &aliasResp); err != nil { return err } + + if aliasResp.AliasExists { + roomID, err = r.DB.GetRoomIDForAlias(ctx, request.Alias) + if err != nil { + return err + } + } } - } + */ response.RoomID = roomID return nil } -// GetAliasesForRoomID implements alias.RoomserverAliasAPI -func (r *RoomserverAliasAPI) GetAliasesForRoomID( +// GetAliasesForRoomID implements alias.RoomserverInternalAPI +func (r *RoomserverInternalAPI) GetAliasesForRoomID( ctx context.Context, - request *roomserverAPI.GetAliasesForRoomIDRequest, - response *roomserverAPI.GetAliasesForRoomIDResponse, + request *api.GetAliasesForRoomIDRequest, + response *api.GetAliasesForRoomIDResponse, ) error { // Look up the aliases in the database for the given RoomID aliases, err := r.DB.GetAliasesForRoomID(ctx, request.RoomID) @@ -140,11 +132,11 @@ func (r *RoomserverAliasAPI) GetAliasesForRoomID( return nil } -// GetCreatorIDForAlias implements alias.RoomserverAliasAPI -func (r *RoomserverAliasAPI) GetCreatorIDForAlias( +// GetCreatorIDForAlias implements alias.RoomserverInternalAPI +func (r *RoomserverInternalAPI) GetCreatorIDForAlias( ctx context.Context, - request *roomserverAPI.GetCreatorIDForAliasRequest, - response *roomserverAPI.GetCreatorIDForAliasResponse, + request *api.GetCreatorIDForAliasRequest, + response *api.GetCreatorIDForAliasResponse, ) error { // Look up the aliases in the database for the given RoomID creatorID, err := r.DB.GetCreatorIDForAlias(ctx, request.Alias) @@ -156,11 +148,11 @@ func (r *RoomserverAliasAPI) GetCreatorIDForAlias( return nil } -// RemoveRoomAlias implements alias.RoomserverAliasAPI -func (r *RoomserverAliasAPI) RemoveRoomAlias( +// RemoveRoomAlias implements alias.RoomserverInternalAPI +func (r *RoomserverInternalAPI) RemoveRoomAlias( ctx context.Context, - request *roomserverAPI.RemoveRoomAliasRequest, - response *roomserverAPI.RemoveRoomAliasResponse, + request *api.RemoveRoomAliasRequest, + response *api.RemoveRoomAliasResponse, ) error { // Look up the room ID in the database roomID, err := r.DB.GetRoomIDForAlias(ctx, request.Alias) @@ -186,7 +178,7 @@ type roomAliasesContent struct { // Build the updated m.room.aliases event to send to the room after addition or // removal of an alias -func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent( +func (r *RoomserverInternalAPI) sendUpdatedAliasesEvent( ctx context.Context, userID string, roomID string, ) error { serverName := string(r.Cfg.Matrix.ServerName) @@ -222,12 +214,12 @@ func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent( if len(eventsNeeded.Tuples()) == 0 { return errors.New("expecting state tuples for event builder, got none") } - req := roomserverAPI.QueryLatestEventsAndStateRequest{ + req := api.QueryLatestEventsAndStateRequest{ RoomID: roomID, StateToFetch: eventsNeeded.Tuples(), } - var res roomserverAPI.QueryLatestEventsAndStateResponse - if err = r.QueryAPI.QueryLatestEventsAndState(ctx, &req, &res); err != nil { + var res api.QueryLatestEventsAndStateResponse + if err = r.QueryLatestEventsAndState(ctx, &req, &res); err != nil { return err } builder.Depth = res.Depth @@ -263,91 +255,17 @@ func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent( } // Create the request - ire := roomserverAPI.InputRoomEvent{ - Kind: roomserverAPI.KindNew, + ire := api.InputRoomEvent{ + Kind: api.KindNew, Event: event.Headered(roomVersion), AuthEventIDs: event.AuthEventIDs(), SendAsServer: serverName, } - inputReq := roomserverAPI.InputRoomEventsRequest{ - InputRoomEvents: []roomserverAPI.InputRoomEvent{ire}, + inputReq := api.InputRoomEventsRequest{ + InputRoomEvents: []api.InputRoomEvent{ire}, } - var inputRes roomserverAPI.InputRoomEventsResponse + var inputRes api.InputRoomEventsResponse // Send the request - return r.InputAPI.InputRoomEvents(ctx, &inputReq, &inputRes) -} - -// SetupHTTP adds the RoomserverAliasAPI handlers to the http.ServeMux. -func (r *RoomserverAliasAPI) SetupHTTP(servMux *http.ServeMux) { - servMux.Handle( - roomserverAPI.RoomserverSetRoomAliasPath, - common.MakeInternalAPI("setRoomAlias", func(req *http.Request) util.JSONResponse { - var request roomserverAPI.SetRoomAliasRequest - var response roomserverAPI.SetRoomAliasResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.SetRoomAlias(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - roomserverAPI.RoomserverGetRoomIDForAliasPath, - common.MakeInternalAPI("GetRoomIDForAlias", func(req *http.Request) util.JSONResponse { - var request roomserverAPI.GetRoomIDForAliasRequest - var response roomserverAPI.GetRoomIDForAliasResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.GetRoomIDForAlias(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - roomserverAPI.RoomserverGetCreatorIDForAliasPath, - common.MakeInternalAPI("GetCreatorIDForAlias", func(req *http.Request) util.JSONResponse { - var request roomserverAPI.GetCreatorIDForAliasRequest - var response roomserverAPI.GetCreatorIDForAliasResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.GetCreatorIDForAlias(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - roomserverAPI.RoomserverGetAliasesForRoomIDPath, - common.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse { - var request roomserverAPI.GetAliasesForRoomIDRequest - var response roomserverAPI.GetAliasesForRoomIDResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.GetAliasesForRoomID(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - roomserverAPI.RoomserverRemoveRoomAliasPath, - common.MakeInternalAPI("removeRoomAlias", func(req *http.Request) util.JSONResponse { - var request roomserverAPI.RemoveRoomAliasRequest - var response roomserverAPI.RemoveRoomAliasResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.RemoveRoomAlias(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) + return r.InputRoomEvents(ctx, &inputReq, &inputRes) } diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go new file mode 100644 index 000000000..1dc985efa --- /dev/null +++ b/roomserver/internal/api.go @@ -0,0 +1,300 @@ +package internal + +import ( + "encoding/json" + "net/http" + "sync" + + "github.com/Shopify/sarama" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/common/caching" + "github.com/matrix-org/dendrite/common/config" + fsAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/storage" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" +) + +// RoomserverInternalAPI is an implementation of api.RoomserverInternalAPI +type RoomserverInternalAPI struct { + DB storage.Database + Cfg *config.Dendrite + Producer sarama.SyncProducer + ImmutableCache caching.ImmutableCache + ServerName gomatrixserverlib.ServerName + KeyRing gomatrixserverlib.JSONVerifier + FedClient *gomatrixserverlib.FederationClient + OutputRoomEventTopic string // Kafka topic for new output room events + mutex sync.Mutex // Protects calls to processRoomEvent + fsAPI fsAPI.FederationSenderInternalAPI +} + +// SetupHTTP adds the RoomserverInternalAPI handlers to the http.ServeMux. +// nolint: gocyclo +func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { + servMux.Handle(api.RoomserverInputRoomEventsPath, + common.MakeInternalAPI("inputRoomEvents", func(req *http.Request) util.JSONResponse { + var request api.InputRoomEventsRequest + var response api.InputRoomEventsResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := r.InputRoomEvents(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle(api.RoomserverPerformJoinPath, + common.MakeInternalAPI("performJoin", func(req *http.Request) util.JSONResponse { + var request api.PerformJoinRequest + var response api.PerformJoinResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := r.PerformJoin(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryLatestEventsAndStatePath, + common.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse { + var request api.QueryLatestEventsAndStateRequest + var response api.QueryLatestEventsAndStateResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryLatestEventsAndState(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryStateAfterEventsPath, + common.MakeInternalAPI("queryStateAfterEvents", func(req *http.Request) util.JSONResponse { + var request api.QueryStateAfterEventsRequest + var response api.QueryStateAfterEventsResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryStateAfterEvents(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryEventsByIDPath, + common.MakeInternalAPI("queryEventsByID", func(req *http.Request) util.JSONResponse { + var request api.QueryEventsByIDRequest + var response api.QueryEventsByIDResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryEventsByID(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryMembershipForUserPath, + common.MakeInternalAPI("QueryMembershipForUser", func(req *http.Request) util.JSONResponse { + var request api.QueryMembershipForUserRequest + var response api.QueryMembershipForUserResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryMembershipForUser(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryMembershipsForRoomPath, + common.MakeInternalAPI("queryMembershipsForRoom", func(req *http.Request) util.JSONResponse { + var request api.QueryMembershipsForRoomRequest + var response api.QueryMembershipsForRoomResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryMembershipsForRoom(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryInvitesForUserPath, + common.MakeInternalAPI("queryInvitesForUser", func(req *http.Request) util.JSONResponse { + var request api.QueryInvitesForUserRequest + var response api.QueryInvitesForUserResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryInvitesForUser(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryServerAllowedToSeeEventPath, + common.MakeInternalAPI("queryServerAllowedToSeeEvent", func(req *http.Request) util.JSONResponse { + var request api.QueryServerAllowedToSeeEventRequest + var response api.QueryServerAllowedToSeeEventResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryServerAllowedToSeeEvent(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryMissingEventsPath, + common.MakeInternalAPI("queryMissingEvents", func(req *http.Request) util.JSONResponse { + var request api.QueryMissingEventsRequest + var response api.QueryMissingEventsResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryMissingEvents(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryStateAndAuthChainPath, + common.MakeInternalAPI("queryStateAndAuthChain", func(req *http.Request) util.JSONResponse { + var request api.QueryStateAndAuthChainRequest + var response api.QueryStateAndAuthChainResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryStateAndAuthChain(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryBackfillPath, + common.MakeInternalAPI("QueryBackfill", func(req *http.Request) util.JSONResponse { + var request api.QueryBackfillRequest + var response api.QueryBackfillResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryBackfill(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryRoomVersionCapabilitiesPath, + common.MakeInternalAPI("QueryRoomVersionCapabilities", func(req *http.Request) util.JSONResponse { + var request api.QueryRoomVersionCapabilitiesRequest + var response api.QueryRoomVersionCapabilitiesResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryRoomVersionCapabilities(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryRoomVersionForRoomPath, + common.MakeInternalAPI("QueryRoomVersionForRoom", func(req *http.Request) util.JSONResponse { + var request api.QueryRoomVersionForRoomRequest + var response api.QueryRoomVersionForRoomResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryRoomVersionForRoom(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverSetRoomAliasPath, + common.MakeInternalAPI("setRoomAlias", func(req *http.Request) util.JSONResponse { + var request api.SetRoomAliasRequest + var response api.SetRoomAliasResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.SetRoomAlias(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverGetRoomIDForAliasPath, + common.MakeInternalAPI("GetRoomIDForAlias", func(req *http.Request) util.JSONResponse { + var request api.GetRoomIDForAliasRequest + var response api.GetRoomIDForAliasResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.GetRoomIDForAlias(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverGetCreatorIDForAliasPath, + common.MakeInternalAPI("GetCreatorIDForAlias", func(req *http.Request) util.JSONResponse { + var request api.GetCreatorIDForAliasRequest + var response api.GetCreatorIDForAliasResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.GetCreatorIDForAlias(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverGetAliasesForRoomIDPath, + common.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse { + var request api.GetAliasesForRoomIDRequest + var response api.GetAliasesForRoomIDResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.GetAliasesForRoomID(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverRemoveRoomAliasPath, + common.MakeInternalAPI("removeRoomAlias", func(req *http.Request) util.JSONResponse { + var request api.RemoveRoomAliasRequest + var response api.RemoveRoomAliasResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.RemoveRoomAlias(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) +} diff --git a/roomserver/input/input.go b/roomserver/internal/input.go similarity index 55% rename from roomserver/input/input.go rename to roomserver/internal/input.go index bd029d8df..19ebea660 100644 --- a/roomserver/input/input.go +++ b/roomserver/internal/input.go @@ -13,33 +13,27 @@ // limitations under the License. // Package input contains the code processes new room events -package input +package internal import ( "context" "encoding/json" - "net/http" - "sync" - "github.com/matrix-org/dendrite/common" + "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/util" - sarama "gopkg.in/Shopify/sarama.v1" + + fsAPI "github.com/matrix-org/dendrite/federationsender/api" ) -// RoomserverInputAPI implements api.RoomserverInputAPI -type RoomserverInputAPI struct { - DB RoomEventDatabase - Producer sarama.SyncProducer - // The kafkaesque topic to output new room events to. - // This is the name used in kafka to identify the stream to write events to. - OutputRoomEventTopic string - // Protects calls to processRoomEvent - mutex sync.Mutex +// SetFederationSenderInputAPI passes in a federation sender input API reference +// so that we can avoid the chicken-and-egg problem of both the roomserver input API +// and the federation sender input API being interdependent. +func (r *RoomserverInternalAPI) SetFederationSenderAPI(fsAPI fsAPI.FederationSenderInternalAPI) { + r.fsAPI = fsAPI } // WriteOutputEvents implements OutputRoomEventWriter -func (r *RoomserverInputAPI) WriteOutputEvents(roomID string, updates []api.OutputEvent) error { +func (r *RoomserverInternalAPI) WriteOutputEvents(roomID string, updates []api.OutputEvent) error { messages := make([]*sarama.ProducerMessage, len(updates)) for i := range updates { value, err := json.Marshal(updates[i]) @@ -55,8 +49,8 @@ func (r *RoomserverInputAPI) WriteOutputEvents(roomID string, updates []api.Outp return r.Producer.SendMessages(messages) } -// InputRoomEvents implements api.RoomserverInputAPI -func (r *RoomserverInputAPI) InputRoomEvents( +// InputRoomEvents implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) InputRoomEvents( ctx context.Context, request *api.InputRoomEventsRequest, response *api.InputRoomEventsResponse, @@ -76,20 +70,3 @@ func (r *RoomserverInputAPI) InputRoomEvents( } return nil } - -// SetupHTTP adds the RoomserverInputAPI handlers to the http.ServeMux. -func (r *RoomserverInputAPI) SetupHTTP(servMux *http.ServeMux) { - servMux.Handle(api.RoomserverInputRoomEventsPath, - common.MakeInternalAPI("inputRoomEvents", func(req *http.Request) util.JSONResponse { - var request api.InputRoomEventsRequest - var response api.InputRoomEventsResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.MessageResponse(http.StatusBadRequest, err.Error()) - } - if err := r.InputRoomEvents(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) -} diff --git a/roomserver/input/authevents.go b/roomserver/internal/input_authevents.go similarity index 98% rename from roomserver/input/authevents.go rename to roomserver/internal/input_authevents.go index 456a01c79..e3828f566 100644 --- a/roomserver/input/authevents.go +++ b/roomserver/internal/input_authevents.go @@ -12,12 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -package input +package internal import ( "context" "sort" + "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -26,7 +27,7 @@ import ( // Returns the numeric IDs for the auth events. func checkAuthEvents( ctx context.Context, - db RoomEventDatabase, + db storage.Database, event gomatrixserverlib.HeaderedEvent, authEventIDs []string, ) ([]types.EventNID, error) { @@ -127,7 +128,7 @@ func (ae *authEvents) lookupEvent(typeNID types.EventTypeNID, stateKey string) * // loadAuthEvents loads the events needed for authentication from the supplied room state. func loadAuthEvents( ctx context.Context, - db RoomEventDatabase, + db storage.Database, needed gomatrixserverlib.StateNeeded, state []types.StateEntry, ) (result authEvents, err error) { diff --git a/roomserver/input/authevents_test.go b/roomserver/internal/input_authevents_test.go similarity index 99% rename from roomserver/input/authevents_test.go rename to roomserver/internal/input_authevents_test.go index 0621a0842..6b981571b 100644 --- a/roomserver/input/authevents_test.go +++ b/roomserver/internal/input_authevents_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package input +package internal import ( "testing" diff --git a/roomserver/input/events.go b/roomserver/internal/input_events.go similarity index 72% rename from roomserver/input/events.go rename to roomserver/internal/input_events.go index 2bb0d0a05..6da63716c 100644 --- a/roomserver/input/events.go +++ b/roomserver/internal/input_events.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package input +package internal import ( "context" @@ -23,63 +23,13 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/state" - "github.com/matrix-org/dendrite/roomserver/state/database" + "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" ) -// A RoomEventDatabase has the storage APIs needed to store a room event. -type RoomEventDatabase interface { - database.RoomStateDatabase - // Stores a matrix room event in the database - StoreEvent( - ctx context.Context, - event gomatrixserverlib.Event, - txnAndSessionID *api.TransactionID, - authEventNIDs []types.EventNID, - ) (types.RoomNID, types.StateAtEvent, error) - // Look up the state entries for a list of string event IDs - // Returns an error if the there is an error talking to the database - // Returns a types.MissingEventError if the event IDs aren't in the database. - StateEntriesForEventIDs( - ctx context.Context, eventIDs []string, - ) ([]types.StateEntry, error) - // Set the state at an event. - SetState( - ctx context.Context, - eventNID types.EventNID, - stateNID types.StateSnapshotNID, - ) error - // Look up the latest events in a room in preparation for an update. - // The RoomRecentEventsUpdater must have Commit or Rollback called on it if this doesn't return an error. - // Returns the latest events in the room and the last eventID sent to the log along with an updater. - // If this returns an error then no further action is required. - GetLatestEventsForUpdate( - ctx context.Context, roomNID types.RoomNID, - ) (updater types.RoomRecentEventsUpdater, err error) - // Look up the string event IDs for a list of numeric event IDs - EventIDs( - ctx context.Context, eventNIDs []types.EventNID, - ) (map[types.EventNID]string, error) - // Build a membership updater for the target user in a room. - MembershipUpdater( - ctx context.Context, roomID, targerUserID string, - roomVersion gomatrixserverlib.RoomVersion, - ) (types.MembershipUpdater, error) - // Look up event ID by transaction's info. - // This is used to determine if the room event is processed/processing already. - // Returns an empty string if no such event exists. - GetTransactionEventID( - ctx context.Context, transactionID string, - sessionID int64, userID string, - ) (string, error) - // Look up the room version for a given room. - GetRoomVersionForRoom( - ctx context.Context, roomID string, - ) (gomatrixserverlib.RoomVersion, error) -} - // OutputRoomEventWriter has the APIs needed to write an event to the output logs. type OutputRoomEventWriter interface { // Write a list of events for a room @@ -93,7 +43,7 @@ type OutputRoomEventWriter interface { // state deltas when sending to kafka streams func processRoomEvent( ctx context.Context, - db RoomEventDatabase, + db storage.Database, ow OutputRoomEventWriter, input api.InputRoomEvent, ) (eventID string, err error) { @@ -104,6 +54,7 @@ func processRoomEvent( // Check that the event passes authentication checks and work out the numeric IDs for the auth events. authEventNIDs, err := checkAuthEvents(ctx, db, headered, input.AuthEventIDs) if err != nil { + logrus.WithError(err).WithField("event_id", event.EventID()).Error("processRoomEvent.checkAuthEvents failed for event") return } @@ -128,6 +79,7 @@ func processRoomEvent( // For outliers we can stop after we've stored the event itself as it // doesn't have any associated state to store and we don't need to // notify anyone about it. + logrus.WithField("event_id", event.EventID()).WithField("type", event.Type()).WithField("room", event.RoomID()).Info("Stored outlier") return event.EventID(), nil } @@ -140,11 +92,6 @@ func processRoomEvent( } } - if input.Kind == api.KindBackfill { - // Backfill is not implemented. - panic("Not implemented") - } - // Update the extremities of the event graph for the room return event.EventID(), updateLatestEvents( ctx, db, ow, roomNID, stateAtEvent, event, input.SendAsServer, input.TransactionID, @@ -153,7 +100,7 @@ func processRoomEvent( func calculateAndSetState( ctx context.Context, - db RoomEventDatabase, + db storage.Database, input api.InputRoomEvent, roomNID types.RoomNID, stateAtEvent *types.StateAtEvent, @@ -184,7 +131,7 @@ func calculateAndSetState( func processInviteEvent( ctx context.Context, - db RoomEventDatabase, + db storage.Database, ow OutputRoomEventWriter, input api.InputInviteEvent, ) (err error) { @@ -247,8 +194,23 @@ func processInviteEvent( event := input.Event.Unwrap() - if err = event.SetUnsignedField("invite_room_state", input.InviteRoomState); err != nil { - return err + if len(input.InviteRoomState) > 0 { + // If we were supplied with some invite room state already (which is + // most likely to be if the event came in over federation) then use + // that. + if err = event.SetUnsignedField("invite_room_state", input.InviteRoomState); err != nil { + return err + } + } else { + // There's no invite room state, so let's have a go at building it + // up from local data (which is most likely to be if the event came + // from the CS API). If we know about the room then we can insert + // the invite room state, if we don't then we just fail quietly. + if irs, ierr := buildInviteStrippedState(ctx, db, input); ierr == nil { + if err = event.SetUnsignedField("invite_room_state", irs); err != nil { + return err + } + } } outputUpdates, err := updateToInviteMembership(updater, &event, nil, input.Event.RoomVersion) @@ -263,3 +225,50 @@ func processInviteEvent( succeeded = true return nil } + +func buildInviteStrippedState( + ctx context.Context, + db storage.Database, + input api.InputInviteEvent, +) ([]gomatrixserverlib.InviteV2StrippedState, error) { + roomNID, err := db.RoomNID(ctx, input.Event.RoomID()) + if err != nil || roomNID == 0 { + return nil, fmt.Errorf("room %q unknown", input.Event.RoomID()) + } + stateWanted := []gomatrixserverlib.StateKeyTuple{} + for _, t := range []string{ + gomatrixserverlib.MRoomName, gomatrixserverlib.MRoomCanonicalAlias, + gomatrixserverlib.MRoomAliases, gomatrixserverlib.MRoomJoinRules, + } { + stateWanted = append(stateWanted, gomatrixserverlib.StateKeyTuple{ + EventType: t, + StateKey: "", + }) + } + _, currentStateSnapshotNID, _, err := db.LatestEventIDs(ctx, roomNID) + if err != nil { + return nil, err + } + roomState := state.NewStateResolution(db) + stateEntries, err := roomState.LoadStateAtSnapshotForStringTuples( + ctx, currentStateSnapshotNID, stateWanted, + ) + if err != nil { + return nil, err + } + stateNIDs := []types.EventNID{} + for _, stateNID := range stateEntries { + stateNIDs = append(stateNIDs, stateNID.EventNID) + } + stateEvents, err := db.Events(ctx, stateNIDs) + if err != nil { + return nil, err + } + inviteState := []gomatrixserverlib.InviteV2StrippedState{ + gomatrixserverlib.NewInviteV2StrippedState(&input.Event.Event), + } + for _, event := range stateEvents { + inviteState = append(inviteState, gomatrixserverlib.NewInviteV2StrippedState(&event.Event)) + } + return inviteState, nil +} diff --git a/roomserver/input/latest_events.go b/roomserver/internal/input_latest_events.go similarity index 98% rename from roomserver/input/latest_events.go rename to roomserver/internal/input_latest_events.go index 525a6f518..42be0f408 100644 --- a/roomserver/input/latest_events.go +++ b/roomserver/internal/input_latest_events.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package input +package internal import ( "bytes" @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/state" + "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -47,7 +48,7 @@ import ( // Can only be called once at a time func updateLatestEvents( ctx context.Context, - db RoomEventDatabase, + db storage.Database, ow OutputRoomEventWriter, roomNID types.RoomNID, stateAtEvent types.StateAtEvent, @@ -86,7 +87,7 @@ func updateLatestEvents( // when there are so many variables to pass around. type latestEventsUpdater struct { ctx context.Context - db RoomEventDatabase + db storage.Database updater types.RoomRecentEventsUpdater ow OutputRoomEventWriter roomNID types.RoomNID diff --git a/roomserver/input/membership.go b/roomserver/internal/input_membership.go similarity index 98% rename from roomserver/input/membership.go rename to roomserver/internal/input_membership.go index ee39ff5eb..cba75b4fc 100644 --- a/roomserver/input/membership.go +++ b/roomserver/internal/input_membership.go @@ -12,13 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package input +package internal import ( "context" "fmt" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -29,7 +30,7 @@ import ( // consumers about the invites added or retired by the change in current state. func updateMemberships( ctx context.Context, - db RoomEventDatabase, + db storage.Database, updater types.RoomRecentEventsUpdater, removed, added []types.StateEntry, ) ([]api.OutputEvent, error) { @@ -143,7 +144,7 @@ func updateToInviteMembership( // consider a single stream of events when determining whether a user // is invited, rather than having to combine multiple streams themselves. onie := api.OutputNewInviteEvent{ - Event: (*add).Headered(roomVersion), + Event: add.Headered(roomVersion), RoomVersion: roomVersion, } updates = append(updates, api.OutputEvent{ diff --git a/roomserver/internal/perform_join.go b/roomserver/internal/perform_join.go new file mode 100644 index 000000000..3dfa118fd --- /dev/null +++ b/roomserver/internal/perform_join.go @@ -0,0 +1,199 @@ +package internal + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/matrix-org/dendrite/common" + fsAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" +) + +// WriteOutputEvents implements OutputRoomEventWriter +func (r *RoomserverInternalAPI) PerformJoin( + ctx context.Context, + req *api.PerformJoinRequest, + res *api.PerformJoinResponse, +) error { + _, domain, err := gomatrixserverlib.SplitID('@', req.UserID) + if err != nil { + return fmt.Errorf("Supplied user ID %q in incorrect format", req.UserID) + } + if domain != r.Cfg.Matrix.ServerName { + return fmt.Errorf("User %q does not belong to this homeserver", req.UserID) + } + if strings.HasPrefix(req.RoomIDOrAlias, "!") { + return r.performJoinRoomByID(ctx, req, res) + } + if strings.HasPrefix(req.RoomIDOrAlias, "#") { + return r.performJoinRoomByAlias(ctx, req, res) + } + return fmt.Errorf("Room ID or alias %q is invalid", req.RoomIDOrAlias) +} + +func (r *RoomserverInternalAPI) performJoinRoomByAlias( + ctx context.Context, + req *api.PerformJoinRequest, + res *api.PerformJoinResponse, +) error { + // Get the domain part of the room alias. + _, domain, err := gomatrixserverlib.SplitID('#', req.RoomIDOrAlias) + if err != nil { + return fmt.Errorf("Alias %q is not in the correct format", req.RoomIDOrAlias) + } + req.ServerNames = append(req.ServerNames, domain) + + // Check if this alias matches our own server configuration. If it + // doesn't then we'll need to try a federated join. + var roomID string + if domain != r.Cfg.Matrix.ServerName { + // The alias isn't owned by us, so we will need to try joining using + // a remote server. + dirReq := fsAPI.PerformDirectoryLookupRequest{ + RoomAlias: req.RoomIDOrAlias, // the room alias to lookup + ServerName: domain, // the server to ask + } + dirRes := fsAPI.PerformDirectoryLookupResponse{} + err = r.fsAPI.PerformDirectoryLookup(ctx, &dirReq, &dirRes) + if err != nil { + logrus.WithError(err).Errorf("error looking up alias %q", req.RoomIDOrAlias) + return fmt.Errorf("Looking up alias %q over federation failed: %w", req.RoomIDOrAlias, err) + } + roomID = dirRes.RoomID + req.ServerNames = append(req.ServerNames, dirRes.ServerNames...) + } else { + // Otherwise, look up if we know this room alias locally. + roomID, err = r.DB.GetRoomIDForAlias(ctx, req.RoomIDOrAlias) + if err != nil { + return fmt.Errorf("Lookup room alias %q failed: %w", req.RoomIDOrAlias, err) + } + } + + // If the room ID is empty then we failed to look up the alias. + if roomID == "" { + return fmt.Errorf("Alias %q not found", req.RoomIDOrAlias) + } + + // If we do, then pluck out the room ID and continue the join. + req.RoomIDOrAlias = roomID + return r.performJoinRoomByID(ctx, req, res) +} + +// TODO: Break this function up a bit +// nolint:gocyclo +func (r *RoomserverInternalAPI) performJoinRoomByID( + ctx context.Context, + req *api.PerformJoinRequest, + res *api.PerformJoinResponse, // nolint:unparam +) error { + // Get the domain part of the room ID. + _, domain, err := gomatrixserverlib.SplitID('!', req.RoomIDOrAlias) + if err != nil { + return fmt.Errorf("Room ID %q is invalid", req.RoomIDOrAlias) + } + req.ServerNames = append(req.ServerNames, domain) + + // Prepare the template for the join event. + userID := req.UserID + eb := gomatrixserverlib.EventBuilder{ + Type: gomatrixserverlib.MRoomMember, + Sender: userID, + StateKey: &userID, + RoomID: req.RoomIDOrAlias, + Redacts: "", + } + if err = eb.SetUnsigned(struct{}{}); err != nil { + return fmt.Errorf("eb.SetUnsigned: %w", err) + } + + // It is possible for the request to include some "content" for the + // event. We'll always overwrite the "membership" key, but the rest, + // like "display_name" or "avatar_url", will be kept if supplied. + if req.Content == nil { + req.Content = map[string]interface{}{} + } + req.Content["membership"] = "join" + if err = eb.SetContent(req.Content); err != nil { + return fmt.Errorf("eb.SetContent: %w", err) + } + + // Try to construct an actual join event from the template. + // If this succeeds then it is a sign that the room already exists + // locally on the homeserver. + // TODO: Check what happens if the room exists on the server + // but everyone has since left. I suspect it does the wrong thing. + buildRes := api.QueryLatestEventsAndStateResponse{} + event, err := common.BuildEvent( + ctx, // the request context + &eb, // the template join event + r.Cfg, // the server configuration + time.Now(), // the event timestamp to use + r, // the roomserver API to use + &buildRes, // the query response + ) + + switch err { + case nil: + // The room join is local. Send the new join event into the + // roomserver. First of all check that the user isn't already + // a member of the room. + alreadyJoined := false + for _, se := range buildRes.StateEvents { + if membership, merr := se.Membership(); merr == nil { + if se.StateKey() != nil && *se.StateKey() == *event.StateKey() { + alreadyJoined = (membership == "join") + break + } + } + } + + // If we haven't already joined the room then send an event + // into the room changing our membership status. + if !alreadyJoined { + inputReq := api.InputRoomEventsRequest{ + InputRoomEvents: []api.InputRoomEvent{ + api.InputRoomEvent{ + Kind: api.KindNew, + Event: event.Headered(buildRes.RoomVersion), + AuthEventIDs: event.AuthEventIDs(), + SendAsServer: string(r.Cfg.Matrix.ServerName), + }, + }, + } + inputRes := api.InputRoomEventsResponse{} + if err = r.InputRoomEvents(ctx, &inputReq, &inputRes); err != nil { + return fmt.Errorf("r.InputRoomEvents: %w", err) + } + } + + case common.ErrRoomNoExists: + // The room doesn't exist. First of all check if the room is a local + // room. If it is then there's nothing more to do - the room just + // hasn't been created yet. + if domain == r.Cfg.Matrix.ServerName { + return fmt.Errorf("Room ID %q does not exist", req.RoomIDOrAlias) + } + + // Try joining by all of the supplied server names. + fedReq := fsAPI.PerformJoinRequest{ + RoomID: req.RoomIDOrAlias, // the room ID to try and join + UserID: req.UserID, // the user ID joining the room + ServerNames: req.ServerNames, // the server to try joining with + Content: req.Content, // the membership event content + } + fedRes := fsAPI.PerformJoinResponse{} + err = r.fsAPI.PerformJoin(ctx, &fedReq, &fedRes) + if err != nil { + return fmt.Errorf("Error joining federated room: %q", err) + } + + default: + return fmt.Errorf("Error joining room %q: %w", req.RoomIDOrAlias, err) + } + + return nil +} diff --git a/roomserver/query/query.go b/roomserver/internal/query.go similarity index 57% rename from roomserver/query/query.go rename to roomserver/internal/query.go index 224d9fa22..98adc24b3 100644 --- a/roomserver/query/query.go +++ b/roomserver/internal/query.go @@ -14,96 +14,25 @@ // See the License for the specific language governing permissions and // limitations under the License. -package query +package internal import ( "context" - "encoding/json" - "net/http" + "fmt" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/caching" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/auth" "github.com/matrix-org/dendrite/roomserver/state" - "github.com/matrix-org/dendrite/roomserver/state/database" + "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" + "github.com/sirupsen/logrus" ) -// RoomserverQueryAPIEventDB has a convenience API to fetch events directly by -// EventIDs. -type RoomserverQueryAPIEventDB interface { - // Look up the Events for a list of event IDs. Does not error if event was - // not found. - // Returns an error if the retrieval went wrong. - EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) -} - -// RoomserverQueryAPIDatabase has the storage APIs needed to implement the query API. -type RoomserverQueryAPIDatabase interface { - database.RoomStateDatabase - RoomserverQueryAPIEventDB - // Look up the numeric ID for the room. - // Returns 0 if the room doesn't exists. - // Returns an error if there was a problem talking to the database. - RoomNID(ctx context.Context, roomID string) (types.RoomNID, error) - // Look up event references for the latest events in the room and the current state snapshot. - // Returns the latest events, the current state and the maximum depth of the latest events plus 1. - // Returns an error if there was a problem talking to the database. - LatestEventIDs( - ctx context.Context, roomNID types.RoomNID, - ) ([]gomatrixserverlib.EventReference, types.StateSnapshotNID, int64, error) - // Look up the numeric IDs for a list of events. - // Returns an error if there was a problem talking to the database. - EventNIDs(ctx context.Context, eventIDs []string) (map[string]types.EventNID, error) - // Lookup the event IDs for a batch of event numeric IDs. - // Returns an error if the retrieval went wrong. - EventIDs(ctx context.Context, eventNIDs []types.EventNID) (map[types.EventNID]string, error) - // Lookup the membership of a given user in a given room. - // Returns the numeric ID of the latest membership event sent from this user - // in this room, along a boolean set to true if the user is still in this room, - // false if not. - // Returns an error if there was a problem talking to the database. - GetMembership( - ctx context.Context, roomNID types.RoomNID, requestSenderUserID string, - ) (membershipEventNID types.EventNID, stillInRoom bool, err error) - // Lookup the membership event numeric IDs for all user that are or have - // been members of a given room. Only lookup events of "join" membership if - // joinOnly is set to true. - // Returns an error if there was a problem talking to the database. - GetMembershipEventNIDsForRoom( - ctx context.Context, roomNID types.RoomNID, joinOnly bool, - ) ([]types.EventNID, error) - // Look up the active invites targeting a user in a room and return the - // numeric state key IDs for the user IDs who sent them. - // Returns an error if there was a problem talking to the database. - GetInvitesForUser( - ctx context.Context, - roomNID types.RoomNID, - targetUserNID types.EventStateKeyNID, - ) (senderUserNIDs []types.EventStateKeyNID, err error) - // Look up the string event state keys for a list of numeric event state keys - // Returns an error if there was a problem talking to the database. - EventStateKeys( - context.Context, []types.EventStateKeyNID, - ) (map[types.EventStateKeyNID]string, error) - // Look up the room version for a given room. - GetRoomVersionForRoom( - ctx context.Context, roomID string, - ) (gomatrixserverlib.RoomVersion, error) -} - -// RoomserverQueryAPI is an implementation of api.RoomserverQueryAPI -type RoomserverQueryAPI struct { - DB RoomserverQueryAPIDatabase - ImmutableCache caching.ImmutableCache -} - -// QueryLatestEventsAndState implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryLatestEventsAndState( +// QueryLatestEventsAndState implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryLatestEventsAndState( ctx context.Context, request *api.QueryLatestEventsAndStateRequest, response *api.QueryLatestEventsAndStateResponse, @@ -117,7 +46,7 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( roomState := state.NewStateResolution(r.DB) response.QueryLatestEventsAndStateRequest = *request - roomNID, err := r.DB.RoomNID(ctx, request.RoomID) + roomNID, err := r.DB.RoomNIDExcludingStubs(ctx, request.RoomID) if err != nil { return err } @@ -162,8 +91,8 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( return nil } -// QueryStateAfterEvents implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryStateAfterEvents( +// QueryStateAfterEvents implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryStateAfterEvents( ctx context.Context, request *api.QueryStateAfterEventsRequest, response *api.QueryStateAfterEventsResponse, @@ -177,7 +106,7 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( roomState := state.NewStateResolution(r.DB) response.QueryStateAfterEventsRequest = *request - roomNID, err := r.DB.RoomNID(ctx, request.RoomID) + roomNID, err := r.DB.RoomNIDExcludingStubs(ctx, request.RoomID) if err != nil { return err } @@ -218,8 +147,8 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( return nil } -// QueryEventsByID implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryEventsByID( +// QueryEventsByID implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryEventsByID( ctx context.Context, request *api.QueryEventsByIDRequest, response *api.QueryEventsByIDResponse, @@ -253,7 +182,7 @@ func (r *RoomserverQueryAPI) QueryEventsByID( return nil } -func (r *RoomserverQueryAPI) loadStateEvents( +func (r *RoomserverInternalAPI) loadStateEvents( ctx context.Context, stateEntries []types.StateEntry, ) ([]gomatrixserverlib.Event, error) { eventNIDs := make([]types.EventNID, len(stateEntries)) @@ -263,7 +192,7 @@ func (r *RoomserverQueryAPI) loadStateEvents( return r.loadEvents(ctx, eventNIDs) } -func (r *RoomserverQueryAPI) loadEvents( +func (r *RoomserverInternalAPI) loadEvents( ctx context.Context, eventNIDs []types.EventNID, ) ([]gomatrixserverlib.Event, error) { stateEvents, err := r.DB.Events(ctx, eventNIDs) @@ -278,8 +207,8 @@ func (r *RoomserverQueryAPI) loadEvents( return result, nil } -// QueryMembershipForUser implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryMembershipForUser( +// QueryMembershipForUser implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryMembershipForUser( ctx context.Context, request *api.QueryMembershipForUserRequest, response *api.QueryMembershipForUserResponse, @@ -309,8 +238,8 @@ func (r *RoomserverQueryAPI) QueryMembershipForUser( return nil } -// QueryMembershipsForRoom implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryMembershipsForRoom( +// QueryMembershipsForRoom implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryMembershipsForRoom( ctx context.Context, request *api.QueryMembershipsForRoomRequest, response *api.QueryMembershipsForRoomResponse, @@ -335,6 +264,7 @@ func (r *RoomserverQueryAPI) QueryMembershipsForRoom( response.JoinEvents = []gomatrixserverlib.ClientEvent{} var events []types.Event + var stateEntries []types.StateEntry if stillInRoom { var eventNIDs []types.EventNID eventNIDs, err = r.DB.GetMembershipEventNIDsForRoom(ctx, roomNID, request.JoinedOnly) @@ -344,7 +274,12 @@ func (r *RoomserverQueryAPI) QueryMembershipsForRoom( events, err = r.DB.Events(ctx, eventNIDs) } else { - events, err = r.getMembershipsBeforeEventNID(ctx, membershipEventNID, request.JoinedOnly) + stateEntries, err = stateBeforeEvent(ctx, r.DB, membershipEventNID) + if err != nil { + logrus.WithField("membership_event_nid", membershipEventNID).WithError(err).Error("failed to load state before event") + return err + } + events, err = getMembershipsAtState(ctx, r.DB, stateEntries, request.JoinedOnly) } if err != nil { @@ -359,32 +294,30 @@ func (r *RoomserverQueryAPI) QueryMembershipsForRoom( return nil } -// getMembershipsBeforeEventNID takes the numeric ID of an event and fetches the state -// of the event's room as it was when this event was fired, then filters the state events to -// only keep the "m.room.member" events with a "join" membership. These events are returned. -// Returns an error if there was an issue fetching the events. -func (r *RoomserverQueryAPI) getMembershipsBeforeEventNID( - ctx context.Context, eventNID types.EventNID, joinedOnly bool, -) ([]types.Event, error) { - roomState := state.NewStateResolution(r.DB) - events := []types.Event{} +func stateBeforeEvent(ctx context.Context, db storage.Database, eventNID types.EventNID) ([]types.StateEntry, error) { + roomState := state.NewStateResolution(db) // Lookup the event NID - eIDs, err := r.DB.EventIDs(ctx, []types.EventNID{eventNID}) + eIDs, err := db.EventIDs(ctx, []types.EventNID{eventNID}) if err != nil { return nil, err } eventIDs := []string{eIDs[eventNID]} - prevState, err := r.DB.StateAtEventIDs(ctx, eventIDs) + prevState, err := db.StateAtEventIDs(ctx, eventIDs) if err != nil { return nil, err } // Fetch the state as it was when this event was fired - stateEntries, err := roomState.LoadCombinedStateAfterEvents(ctx, prevState) - if err != nil { - return nil, err - } + return roomState.LoadCombinedStateAfterEvents(ctx, prevState) +} + +// getMembershipsAtState filters the state events to +// only keep the "m.room.member" events with a "join" membership. These events are returned. +// Returns an error if there was an issue fetching the events. +func getMembershipsAtState( + ctx context.Context, db storage.Database, stateEntries []types.StateEntry, joinedOnly bool, +) ([]types.Event, error) { var eventNIDs []types.EventNID for _, entry := range stateEntries { @@ -395,7 +328,7 @@ func (r *RoomserverQueryAPI) getMembershipsBeforeEventNID( } // Get all of the events in this state - stateEvents, err := r.DB.Events(ctx, eventNIDs) + stateEvents, err := db.Events(ctx, eventNIDs) if err != nil { return nil, err } @@ -405,6 +338,7 @@ func (r *RoomserverQueryAPI) getMembershipsBeforeEventNID( } // Filter the events to only keep the "join" membership events + var events []types.Event for _, event := range stateEvents { membership, err := event.Membership() if err != nil { @@ -419,8 +353,8 @@ func (r *RoomserverQueryAPI) getMembershipsBeforeEventNID( return events, nil } -// QueryInvitesForUser implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryInvitesForUser( +// QueryInvitesForUser implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryInvitesForUser( ctx context.Context, request *api.QueryInvitesForUserRequest, response *api.QueryInvitesForUserResponse, @@ -453,8 +387,8 @@ func (r *RoomserverQueryAPI) QueryInvitesForUser( return nil } -// QueryServerAllowedToSeeEvent implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryServerAllowedToSeeEvent( +// QueryServerAllowedToSeeEvent implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryServerAllowedToSeeEvent( ctx context.Context, request *api.QueryServerAllowedToSeeEventRequest, response *api.QueryServerAllowedToSeeEventResponse, @@ -477,7 +411,7 @@ func (r *RoomserverQueryAPI) QueryServerAllowedToSeeEvent( return } -func (r *RoomserverQueryAPI) checkServerAllowedToSeeEvent( +func (r *RoomserverInternalAPI) checkServerAllowedToSeeEvent( ctx context.Context, eventID string, serverName gomatrixserverlib.ServerName, isServerInRoom bool, ) (bool, error) { roomState := state.NewStateResolution(r.DB) @@ -496,8 +430,8 @@ func (r *RoomserverQueryAPI) checkServerAllowedToSeeEvent( return auth.IsServerAllowed(serverName, isServerInRoom, stateAtEvent), nil } -// QueryMissingEvents implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryMissingEvents( +// QueryMissingEvents implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryMissingEvents( ctx context.Context, request *api.QueryMissingEventsRequest, response *api.QueryMissingEventsResponse, @@ -542,11 +476,18 @@ func (r *RoomserverQueryAPI) QueryMissingEvents( } // QueryBackfill implements api.RoomServerQueryAPI -func (r *RoomserverQueryAPI) QueryBackfill( +func (r *RoomserverInternalAPI) QueryBackfill( ctx context.Context, request *api.QueryBackfillRequest, response *api.QueryBackfillResponse, ) error { + // if we are requesting the backfill then we need to do a federation hit + // TODO: we could be more sensible and fetch as many events we already have then request the rest + // which is what the syncapi does already. + if request.ServerName == r.ServerName { + return r.backfillViaFederation(ctx, request, response) + } + // someone else is requesting the backfill, try to service their request. var err error var front []string @@ -588,7 +529,65 @@ func (r *RoomserverQueryAPI) QueryBackfill( return err } -func (r *RoomserverQueryAPI) isServerCurrentlyInRoom(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID string) (bool, error) { +func (r *RoomserverInternalAPI) backfillViaFederation(ctx context.Context, req *api.QueryBackfillRequest, res *api.QueryBackfillResponse) error { + roomVer, err := r.DB.GetRoomVersionForRoom(ctx, req.RoomID) + if err != nil { + return fmt.Errorf("backfillViaFederation: unknown room version for room %s : %w", req.RoomID, err) + } + requester := newBackfillRequester(r.DB, r.FedClient, r.ServerName) + events, err := gomatrixserverlib.RequestBackfill( + ctx, requester, + r.KeyRing, req.RoomID, roomVer, req.EarliestEventsIDs, req.Limit) + if err != nil { + return err + } + logrus.WithField("room_id", req.RoomID).Infof("backfilled %d events", len(events)) + + // persist these new events - auth checks have already been done + roomNID, backfilledEventMap := persistEvents(ctx, r.DB, events) + if err != nil { + return err + } + + for _, ev := range backfilledEventMap { + // now add state for these events + stateIDs, ok := requester.eventIDToBeforeStateIDs[ev.EventID()] + if !ok { + // this should be impossible as all events returned must have pass Step 5 of the PDU checks + // which requires a list of state IDs. + logrus.WithError(err).WithField("event_id", ev.EventID()).Error("backfillViaFederation: failed to find state IDs for event which passed auth checks") + continue + } + var entries []types.StateEntry + if entries, err = r.DB.StateEntriesForEventIDs(ctx, stateIDs); err != nil { + // attempt to fetch the missing events + r.fetchAndStoreMissingEvents(ctx, roomVer, requester, stateIDs) + // try again + entries, err = r.DB.StateEntriesForEventIDs(ctx, stateIDs) + if err != nil { + logrus.WithError(err).WithField("event_id", ev.EventID()).Error("backfillViaFederation: failed to get state entries for event") + return err + } + } + + var beforeStateSnapshotNID types.StateSnapshotNID + if beforeStateSnapshotNID, err = r.DB.AddState(ctx, roomNID, nil, entries); err != nil { + logrus.WithError(err).WithField("event_id", ev.EventID()).Error("backfillViaFederation: failed to persist state entries to get snapshot nid") + return err + } + util.GetLogger(ctx).Infof("Backfilled event %s (nid=%d) getting snapshot %v with entries %+v", ev.EventID(), ev.EventNID, beforeStateSnapshotNID, entries) + if err = r.DB.SetState(ctx, ev.EventNID, beforeStateSnapshotNID); err != nil { + logrus.WithError(err).WithField("event_id", ev.EventID()).Error("backfillViaFederation: failed to persist snapshot nid") + } + } + + // TODO: update backwards extremities, as that should be moved from syncapi to roomserver at some point. + + res.Events = events + return nil +} + +func (r *RoomserverInternalAPI) isServerCurrentlyInRoom(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID string) (bool, error) { roomNID, err := r.DB.RoomNID(ctx, roomID) if err != nil { return false, err @@ -610,9 +609,69 @@ func (r *RoomserverQueryAPI) isServerCurrentlyInRoom(ctx context.Context, server return auth.IsAnyUserOnServerWithMembership(serverName, gmslEvents, gomatrixserverlib.Join), nil } +// fetchAndStoreMissingEvents does a best-effort fetch and store of missing events specified in stateIDs. Returns no error as it is just +// best effort. +func (r *RoomserverInternalAPI) fetchAndStoreMissingEvents(ctx context.Context, roomVer gomatrixserverlib.RoomVersion, + backfillRequester *backfillRequester, stateIDs []string) { + + servers := backfillRequester.servers + + // work out which are missing + nidMap, err := r.DB.EventNIDs(ctx, stateIDs) + if err != nil { + util.GetLogger(ctx).WithError(err).Warn("cannot query missing events") + return + } + missingMap := make(map[string]*gomatrixserverlib.HeaderedEvent) // id -> event + for _, id := range stateIDs { + if _, ok := nidMap[id]; !ok { + missingMap[id] = nil + } + } + util.GetLogger(ctx).Infof("Fetching %d missing state events (from %d possible servers)", len(missingMap), len(servers)) + + // fetch the events from federation. Loop the servers first so if we find one that works we stick with them + for _, srv := range servers { + for id, ev := range missingMap { + if ev != nil { + continue // already found + } + logger := util.GetLogger(ctx).WithField("server", srv).WithField("event_id", id) + res, err := r.FedClient.GetEvent(ctx, srv, id) + if err != nil { + logger.WithError(err).Warn("failed to get event from server") + continue + } + loader := gomatrixserverlib.NewEventsLoader(roomVer, r.KeyRing, backfillRequester, backfillRequester.ProvideEvents, false) + result, err := loader.LoadAndVerify(ctx, res.PDUs, gomatrixserverlib.TopologicalOrderByPrevEvents) + if err != nil { + logger.WithError(err).Warn("failed to load and verify event") + continue + } + logger.Infof("returned %d PDUs which made events %+v", len(res.PDUs), result) + for _, res := range result { + if res.Error != nil { + logger.WithError(err).Warn("event failed PDU checks") + continue + } + missingMap[id] = res.Event + } + } + } + + var newEvents []gomatrixserverlib.HeaderedEvent + for _, ev := range missingMap { + if ev != nil { + newEvents = append(newEvents, *ev) + } + } + util.GetLogger(ctx).Infof("Persisting %d new events", len(newEvents)) + persistEvents(ctx, r.DB, newEvents) +} + // TODO: Remove this when we have tests to assert correctness of this function // nolint:gocyclo -func (r *RoomserverQueryAPI) scanEventTree( +func (r *RoomserverInternalAPI) scanEventTree( ctx context.Context, front []string, visited map[string]bool, limit int, serverName gomatrixserverlib.ServerName, ) ([]types.EventNID, error) { @@ -705,14 +764,14 @@ BFSLoop: return resultNIDs, err } -// QueryStateAndAuthChain implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryStateAndAuthChain( +// QueryStateAndAuthChain implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryStateAndAuthChain( ctx context.Context, request *api.QueryStateAndAuthChainRequest, response *api.QueryStateAndAuthChainResponse, ) error { response.QueryStateAndAuthChainRequest = *request - roomNID, err := r.DB.RoomNID(ctx, request.RoomID) + roomNID, err := r.DB.RoomNIDExcludingStubs(ctx, request.RoomID) if err != nil { return err } @@ -741,7 +800,7 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( } authEventIDs = util.UniqueStrings(authEventIDs) // de-dupe - authEvents, err := getAuthChain(ctx, r.DB, authEventIDs) + authEvents, err := getAuthChain(ctx, r.DB.EventsFromIDs, authEventIDs) if err != nil { return err } @@ -765,7 +824,7 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( return err } -func (r *RoomserverQueryAPI) loadStateAtEventIDs(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) { +func (r *RoomserverInternalAPI) loadStateAtEventIDs(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) { roomState := state.NewStateResolution(r.DB) prevStates, err := r.DB.StateAtEventIDs(ctx, eventIDs) if err != nil { @@ -788,12 +847,14 @@ func (r *RoomserverQueryAPI) loadStateAtEventIDs(ctx context.Context, eventIDs [ return r.loadStateEvents(ctx, stateEntries) } +type eventsFromIDs func(context.Context, []string) ([]types.Event, error) + // getAuthChain fetches the auth chain for the given auth events. An auth chain // is the list of all events that are referenced in the auth_events section, and // all their auth_events, recursively. The returned set of events contain the // given events. Will *not* error if we don't have all auth events. func getAuthChain( - ctx context.Context, dB RoomserverQueryAPIEventDB, authEventIDs []string, + ctx context.Context, fn eventsFromIDs, authEventIDs []string, ) ([]gomatrixserverlib.Event, error) { // List of event IDs to fetch. On each pass, these events will be requested // from the database and the `eventsToFetch` will be updated with any new @@ -804,7 +865,7 @@ func getAuthChain( for len(eventsToFetch) > 0 { // Try to retrieve the events from the database. - events, err := dB.EventsFromIDs(ctx, eventsToFetch) + events, err := fn(ctx, eventsToFetch) if err != nil { return nil, err } @@ -839,43 +900,37 @@ func getAuthChain( return authEvents, nil } -// QueryServersInRoomAtEvent implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryServersInRoomAtEvent( - ctx context.Context, - request *api.QueryServersInRoomAtEventRequest, - response *api.QueryServersInRoomAtEventResponse, -) error { - // getMembershipsBeforeEventNID requires a NID, so retrieving the NID for - // the event is necessary. - NIDs, err := r.DB.EventNIDs(ctx, []string{request.EventID}) - if err != nil { - return err +func persistEvents(ctx context.Context, db storage.Database, events []gomatrixserverlib.HeaderedEvent) (types.RoomNID, map[string]types.Event) { + var roomNID types.RoomNID + backfilledEventMap := make(map[string]types.Event) + for _, ev := range events { + nidMap, err := db.EventNIDs(ctx, ev.AuthEventIDs()) + if err != nil { // this shouldn't happen as RequestBackfill already found them + logrus.WithError(err).WithField("auth_events", ev.AuthEventIDs()).Error("Failed to find one or more auth events") + continue + } + authNids := make([]types.EventNID, len(nidMap)) + i := 0 + for _, nid := range nidMap { + authNids[i] = nid + i++ + } + var stateAtEvent types.StateAtEvent + roomNID, stateAtEvent, err = db.StoreEvent(ctx, ev.Unwrap(), nil, authNids) + if err != nil { + logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to persist event") + continue + } + backfilledEventMap[ev.EventID()] = types.Event{ + EventNID: stateAtEvent.StateEntry.EventNID, + Event: ev.Unwrap(), + } } - - // Retrieve all "m.room.member" state events of "join" membership, which - // contains the list of users in the room before the event, therefore all - // the servers in it at that moment. - events, err := r.getMembershipsBeforeEventNID(ctx, NIDs[request.EventID], true) - if err != nil { - return err - } - - // Store the server names in a temporary map to avoid duplicates. - servers := make(map[gomatrixserverlib.ServerName]bool) - for _, event := range events { - servers[event.Origin()] = true - } - - // Populate the response. - for server := range servers { - response.Servers = append(response.Servers, server) - } - - return nil + return roomNID, backfilledEventMap } -// QueryRoomVersionCapabilities implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryRoomVersionCapabilities( +// QueryRoomVersionCapabilities implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryRoomVersionCapabilities( ctx context.Context, request *api.QueryRoomVersionCapabilitiesRequest, response *api.QueryRoomVersionCapabilitiesResponse, @@ -892,8 +947,8 @@ func (r *RoomserverQueryAPI) QueryRoomVersionCapabilities( return nil } -// QueryRoomVersionCapabilities implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryRoomVersionForRoom( +// QueryRoomVersionCapabilities implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryRoomVersionForRoom( ctx context.Context, request *api.QueryRoomVersionForRoomRequest, response *api.QueryRoomVersionForRoomResponse, @@ -911,190 +966,3 @@ func (r *RoomserverQueryAPI) QueryRoomVersionForRoom( r.ImmutableCache.StoreRoomVersion(request.RoomID, response.RoomVersion) return nil } - -// SetupHTTP adds the RoomserverQueryAPI handlers to the http.ServeMux. -// nolint: gocyclo -func (r *RoomserverQueryAPI) SetupHTTP(servMux *http.ServeMux) { - servMux.Handle( - api.RoomserverQueryLatestEventsAndStatePath, - common.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse { - var request api.QueryLatestEventsAndStateRequest - var response api.QueryLatestEventsAndStateResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryLatestEventsAndState(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryStateAfterEventsPath, - common.MakeInternalAPI("queryStateAfterEvents", func(req *http.Request) util.JSONResponse { - var request api.QueryStateAfterEventsRequest - var response api.QueryStateAfterEventsResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryStateAfterEvents(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryEventsByIDPath, - common.MakeInternalAPI("queryEventsByID", func(req *http.Request) util.JSONResponse { - var request api.QueryEventsByIDRequest - var response api.QueryEventsByIDResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryEventsByID(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryMembershipForUserPath, - common.MakeInternalAPI("QueryMembershipForUser", func(req *http.Request) util.JSONResponse { - var request api.QueryMembershipForUserRequest - var response api.QueryMembershipForUserResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryMembershipForUser(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryMembershipsForRoomPath, - common.MakeInternalAPI("queryMembershipsForRoom", func(req *http.Request) util.JSONResponse { - var request api.QueryMembershipsForRoomRequest - var response api.QueryMembershipsForRoomResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryMembershipsForRoom(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryInvitesForUserPath, - common.MakeInternalAPI("queryInvitesForUser", func(req *http.Request) util.JSONResponse { - var request api.QueryInvitesForUserRequest - var response api.QueryInvitesForUserResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryInvitesForUser(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryServerAllowedToSeeEventPath, - common.MakeInternalAPI("queryServerAllowedToSeeEvent", func(req *http.Request) util.JSONResponse { - var request api.QueryServerAllowedToSeeEventRequest - var response api.QueryServerAllowedToSeeEventResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryServerAllowedToSeeEvent(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryMissingEventsPath, - common.MakeInternalAPI("queryMissingEvents", func(req *http.Request) util.JSONResponse { - var request api.QueryMissingEventsRequest - var response api.QueryMissingEventsResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryMissingEvents(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryStateAndAuthChainPath, - common.MakeInternalAPI("queryStateAndAuthChain", func(req *http.Request) util.JSONResponse { - var request api.QueryStateAndAuthChainRequest - var response api.QueryStateAndAuthChainResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryStateAndAuthChain(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryBackfillPath, - common.MakeInternalAPI("QueryBackfill", func(req *http.Request) util.JSONResponse { - var request api.QueryBackfillRequest - var response api.QueryBackfillResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryBackfill(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryServersInRoomAtEventPath, - common.MakeInternalAPI("QueryServersInRoomAtEvent", func(req *http.Request) util.JSONResponse { - var request api.QueryServersInRoomAtEventRequest - var response api.QueryServersInRoomAtEventResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryServersInRoomAtEvent(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryRoomVersionCapabilitiesPath, - common.MakeInternalAPI("QueryRoomVersionCapabilities", func(req *http.Request) util.JSONResponse { - var request api.QueryRoomVersionCapabilitiesRequest - var response api.QueryRoomVersionCapabilitiesResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryRoomVersionCapabilities(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryRoomVersionForRoomPath, - common.MakeInternalAPI("QueryRoomVersionForRoom", func(req *http.Request) util.JSONResponse { - var request api.QueryRoomVersionForRoomRequest - var response api.QueryRoomVersionForRoomResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryRoomVersionForRoom(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) -} diff --git a/roomserver/internal/query_backfill.go b/roomserver/internal/query_backfill.go new file mode 100644 index 000000000..d42038e74 --- /dev/null +++ b/roomserver/internal/query_backfill.go @@ -0,0 +1,278 @@ +package internal + +import ( + "context" + + "github.com/matrix-org/dendrite/roomserver/auth" + "github.com/matrix-org/dendrite/roomserver/storage" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" +) + +// backfillRequester implements gomatrixserverlib.BackfillRequester +type backfillRequester struct { + db storage.Database + fedClient *gomatrixserverlib.FederationClient + thisServer gomatrixserverlib.ServerName + + // per-request state + servers []gomatrixserverlib.ServerName + eventIDToBeforeStateIDs map[string][]string + eventIDMap map[string]gomatrixserverlib.Event +} + +func newBackfillRequester(db storage.Database, fedClient *gomatrixserverlib.FederationClient, thisServer gomatrixserverlib.ServerName) *backfillRequester { + return &backfillRequester{ + db: db, + fedClient: fedClient, + thisServer: thisServer, + eventIDToBeforeStateIDs: make(map[string][]string), + eventIDMap: make(map[string]gomatrixserverlib.Event), + } +} + +func (b *backfillRequester) StateIDsBeforeEvent(ctx context.Context, targetEvent gomatrixserverlib.HeaderedEvent) ([]string, error) { + b.eventIDMap[targetEvent.EventID()] = targetEvent.Unwrap() + if ids, ok := b.eventIDToBeforeStateIDs[targetEvent.EventID()]; ok { + return ids, nil + } + // if we have exactly 1 prev event and we know the state of the room at that prev event, then just roll forward the prev event. + // Else, we have to hit /state_ids because either we don't know the state at all at this event (new backwards extremity) or + // we don't know the result of state res to merge forks (2 or more prev_events) + if len(targetEvent.PrevEventIDs()) == 1 { + prevEventID := targetEvent.PrevEventIDs()[0] + prevEvent, ok := b.eventIDMap[prevEventID] + if !ok { + goto FederationHit + } + prevEventStateIDs, ok := b.eventIDToBeforeStateIDs[prevEventID] + if !ok { + goto FederationHit + } + newStateIDs := b.calculateNewStateIDs(targetEvent.Unwrap(), prevEvent, prevEventStateIDs) + if newStateIDs != nil { + b.eventIDToBeforeStateIDs[targetEvent.EventID()] = newStateIDs + return newStateIDs, nil + } + // else we failed to calculate the new state, so fallthrough + } + +FederationHit: + var lastErr error + logrus.WithField("event_id", targetEvent.EventID()).Info("Requesting /state_ids at event") + for _, srv := range b.servers { // hit any valid server + c := gomatrixserverlib.FederatedStateProvider{ + FedClient: b.fedClient, + RememberAuthEvents: false, + Server: srv, + } + res, err := c.StateIDsBeforeEvent(ctx, targetEvent) + if err != nil { + lastErr = err + continue + } + b.eventIDToBeforeStateIDs[targetEvent.EventID()] = res + return res, nil + } + return nil, lastErr +} + +func (b *backfillRequester) calculateNewStateIDs(targetEvent, prevEvent gomatrixserverlib.Event, prevEventStateIDs []string) []string { + newStateIDs := prevEventStateIDs[:] + if prevEvent.StateKey() == nil { + // state is the same as the previous event + b.eventIDToBeforeStateIDs[targetEvent.EventID()] = newStateIDs + return newStateIDs + } + + missingState := false // true if we are missing the info for a state event ID + foundEvent := false // true if we found a (type, state_key) match + // find which state ID to replace, if any + for i, id := range newStateIDs { + ev, ok := b.eventIDMap[id] + if !ok { + missingState = true + continue + } + // The state IDs BEFORE the target event are the state IDs BEFORE the prev_event PLUS the prev_event itself + if ev.Type() == prevEvent.Type() && ev.StateKey() != nil && *ev.StateKey() == *prevEvent.StateKey() { + newStateIDs[i] = prevEvent.EventID() + foundEvent = true + break + } + } + if !foundEvent && !missingState { + // we can be certain that this is new state + newStateIDs = append(newStateIDs, prevEvent.EventID()) + foundEvent = true + } + + if foundEvent { + b.eventIDToBeforeStateIDs[targetEvent.EventID()] = newStateIDs + return newStateIDs + } + return nil +} + +func (b *backfillRequester) StateBeforeEvent(ctx context.Context, roomVer gomatrixserverlib.RoomVersion, + event gomatrixserverlib.HeaderedEvent, eventIDs []string) (map[string]*gomatrixserverlib.Event, error) { + + // try to fetch the events from the database first + events, err := b.ProvideEvents(roomVer, eventIDs) + if err != nil { + // non-fatal, fallthrough + logrus.WithError(err).Info("Failed to fetch events") + } else { + logrus.Infof("Fetched %d/%d events from the database", len(events), len(eventIDs)) + if len(events) == len(eventIDs) { + result := make(map[string]*gomatrixserverlib.Event) + for i := range events { + result[events[i].EventID()] = &events[i] + b.eventIDMap[events[i].EventID()] = events[i] + } + return result, nil + } + } + + c := gomatrixserverlib.FederatedStateProvider{ + FedClient: b.fedClient, + RememberAuthEvents: false, + Server: b.servers[0], + } + result, err := c.StateBeforeEvent(ctx, roomVer, event, eventIDs) + if err != nil { + return nil, err + } + for eventID, ev := range result { + b.eventIDMap[eventID] = *ev + } + return result, nil +} + +// ServersAtEvent is called when trying to determine which server to request from. +// It returns a list of servers which can be queried for backfill requests. These servers +// will be servers that are in the room already. The entries at the beginning are preferred servers +// and will be tried first. An empty list will fail the request. +func (b *backfillRequester) ServersAtEvent(ctx context.Context, roomID, eventID string) (servers []gomatrixserverlib.ServerName) { + // getMembershipsBeforeEventNID requires a NID, so retrieving the NID for + // the event is necessary. + NIDs, err := b.db.EventNIDs(ctx, []string{eventID}) + if err != nil { + logrus.WithField("event_id", eventID).WithError(err).Error("ServersAtEvent: failed to get event NID for event") + return + } + + stateEntries, err := stateBeforeEvent(ctx, b.db, NIDs[eventID]) + if err != nil { + logrus.WithField("event_id", eventID).WithError(err).Error("ServersAtEvent: failed to load state before event") + return + } + + // possibly return all joined servers depending on history visiblity + memberEventsFromVis, err := joinEventsFromHistoryVisibility(ctx, b.db, roomID, stateEntries) + if err != nil { + logrus.WithError(err).Error("ServersAtEvent: failed calculate servers from history visibility rules") + return + } + logrus.Infof("ServersAtEvent including %d current events from history visibility", len(memberEventsFromVis)) + + // Retrieve all "m.room.member" state events of "join" membership, which + // contains the list of users in the room before the event, therefore all + // the servers in it at that moment. + memberEvents, err := getMembershipsAtState(ctx, b.db, stateEntries, true) + if err != nil { + logrus.WithField("event_id", eventID).WithError(err).Error("ServersAtEvent: failed to get memberships before event") + return + } + memberEvents = append(memberEvents, memberEventsFromVis...) + + // Store the server names in a temporary map to avoid duplicates. + serverSet := make(map[gomatrixserverlib.ServerName]bool) + for _, event := range memberEvents { + serverSet[event.Origin()] = true + } + for server := range serverSet { + if server == b.thisServer { + continue + } + servers = append(servers, server) + } + b.servers = servers + return +} + +// Backfill performs a backfill request to the given server. +// https://matrix.org/docs/spec/server_server/latest#get-matrix-federation-v1-backfill-roomid +func (b *backfillRequester) Backfill(ctx context.Context, server gomatrixserverlib.ServerName, roomID string, + fromEventIDs []string, limit int) (*gomatrixserverlib.Transaction, error) { + + tx, err := b.fedClient.Backfill(ctx, server, roomID, limit, fromEventIDs) + return &tx, err +} + +func (b *backfillRequester) ProvideEvents(roomVer gomatrixserverlib.RoomVersion, eventIDs []string) ([]gomatrixserverlib.Event, error) { + ctx := context.Background() + nidMap, err := b.db.EventNIDs(ctx, eventIDs) + if err != nil { + logrus.WithError(err).WithField("event_ids", eventIDs).Error("Failed to find events") + return nil, err + } + eventNIDs := make([]types.EventNID, len(nidMap)) + i := 0 + for _, nid := range nidMap { + eventNIDs[i] = nid + i++ + } + eventsWithNids, err := b.db.Events(ctx, eventNIDs) + if err != nil { + logrus.WithError(err).WithField("event_nids", eventNIDs).Error("Failed to load events") + return nil, err + } + events := make([]gomatrixserverlib.Event, len(eventsWithNids)) + for i := range eventsWithNids { + events[i] = eventsWithNids[i].Event + } + return events, nil +} + +// joinEventsFromHistoryVisibility returns all CURRENTLY joined members if the provided state indicated a 'shared' history visibility. +// TODO: Long term we probably want a history_visibility table which stores eventNID | visibility_enum so we can just +// pull all events and then filter by that table. +func joinEventsFromHistoryVisibility( + ctx context.Context, db storage.Database, roomID string, stateEntries []types.StateEntry) ([]types.Event, error) { + + var eventNIDs []types.EventNID + for _, entry := range stateEntries { + // Filter the events to retrieve to only keep the membership events + if entry.EventTypeNID == types.MRoomHistoryVisibilityNID && entry.EventStateKeyNID == types.EmptyStateKeyNID { + eventNIDs = append(eventNIDs, entry.EventNID) + break + } + } + + // Get all of the events in this state + stateEvents, err := db.Events(ctx, eventNIDs) + if err != nil { + return nil, err + } + events := make([]gomatrixserverlib.Event, len(stateEvents)) + for i := range stateEvents { + events[i] = stateEvents[i].Event + } + visibility := auth.HistoryVisibilityForRoom(events) + if visibility != "shared" { + logrus.Infof("ServersAtEvent history visibility not shared: %s", visibility) + return nil, nil + } + // get joined members + roomNID, err := db.RoomNID(ctx, roomID) + if err != nil { + return nil, err + } + joinEventNIDs, err := db.GetMembershipEventNIDsForRoom(ctx, roomNID, true) + if err != nil { + return nil, err + } + return db.Events(ctx, joinEventNIDs) +} diff --git a/roomserver/query/query_test.go b/roomserver/internal/query_test.go similarity index 92% rename from roomserver/query/query_test.go rename to roomserver/internal/query_test.go index 7e040c6fb..211ab5083 100644 --- a/roomserver/query/query_test.go +++ b/roomserver/internal/query_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package query +package internal import ( "context" @@ -24,7 +24,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -// used to implement RoomserverQueryAPIEventDB to test getAuthChain +// used to implement RoomserverInternalAPIEventDB to test getAuthChain type getEventDB struct { eventMap map[string]gomatrixserverlib.Event } @@ -79,7 +79,7 @@ func (db *getEventDB) addFakeEvents(graph map[string][]string) error { return nil } -// EventsFromIDs implements RoomserverQueryAPIEventDB +// EventsFromIDs implements RoomserverInternalAPIEventDB func (db *getEventDB) EventsFromIDs(ctx context.Context, eventIDs []string) (res []types.Event, err error) { for _, evID := range eventIDs { res = append(res, types.Event{ @@ -106,7 +106,7 @@ func TestGetAuthChainSingle(t *testing.T) { t.Fatalf("Failed to add events to db: %v", err) } - result, err := getAuthChain(context.TODO(), db, []string{"e"}) + result, err := getAuthChain(context.TODO(), db.EventsFromIDs, []string{"e"}) if err != nil { t.Fatalf("getAuthChain failed: %v", err) } @@ -139,7 +139,7 @@ func TestGetAuthChainMultiple(t *testing.T) { t.Fatalf("Failed to add events to db: %v", err) } - result, err := getAuthChain(context.TODO(), db, []string{"e", "f"}) + result, err := getAuthChain(context.TODO(), db.EventsFromIDs, []string{"e", "f"}) if err != nil { t.Fatalf("getAuthChain failed: %v", err) } diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index fa4f20626..450da5bb6 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -18,12 +18,10 @@ import ( "net/http" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" - asQuery "github.com/matrix-org/dendrite/appservice/query" "github.com/matrix-org/dendrite/common/basecomponent" - "github.com/matrix-org/dendrite/roomserver/alias" - "github.com/matrix-org/dendrite/roomserver/input" - "github.com/matrix-org/dendrite/roomserver/query" + "github.com/matrix-org/dendrite/roomserver/internal" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/sirupsen/logrus" ) @@ -34,38 +32,26 @@ import ( // APIs directly instead of having to use HTTP. func SetupRoomServerComponent( base *basecomponent.BaseDendrite, -) (api.RoomserverAliasAPI, api.RoomserverInputAPI, api.RoomserverQueryAPI) { - roomserverDB, err := storage.Open(string(base.Cfg.Database.RoomServer)) + keyRing gomatrixserverlib.JSONVerifier, + fedClient *gomatrixserverlib.FederationClient, +) api.RoomserverInternalAPI { + roomserverDB, err := storage.Open(string(base.Cfg.Database.RoomServer), base.Cfg.DbProperties()) if err != nil { logrus.WithError(err).Panicf("failed to connect to room server db") } - inputAPI := input.RoomserverInputAPI{ + internalAPI := internal.RoomserverInternalAPI{ DB: roomserverDB, + Cfg: base.Cfg, Producer: base.KafkaProducer, OutputRoomEventTopic: string(base.Cfg.Kafka.Topics.OutputRoomEvent), + ImmutableCache: base.ImmutableCache, + ServerName: base.Cfg.Matrix.ServerName, + FedClient: fedClient, + KeyRing: keyRing, } - inputAPI.SetupHTTP(http.DefaultServeMux) + internalAPI.SetupHTTP(http.DefaultServeMux) - queryAPI := query.RoomserverQueryAPI{ - DB: roomserverDB, - ImmutableCache: base.ImmutableCache, - } - - queryAPI.SetupHTTP(http.DefaultServeMux) - - asAPI := asQuery.AppServiceQueryAPI{Cfg: base.Cfg} - - aliasAPI := alias.RoomserverAliasAPI{ - DB: roomserverDB, - Cfg: base.Cfg, - InputAPI: &inputAPI, - QueryAPI: &queryAPI, - AppserviceAPI: &asAPI, - } - - aliasAPI.SetupHTTP(http.DefaultServeMux) - - return &aliasAPI, &inputAPI, &queryAPI + return &internalAPI } diff --git a/roomserver/state/database/database.go b/roomserver/state/database/database.go deleted file mode 100644 index 80f1b14f4..000000000 --- a/roomserver/state/database/database.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// Copyright 2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package database - -import ( - "context" - - "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/gomatrixserverlib" -) - -// A RoomStateDatabase has the storage APIs needed to load state from the database -type RoomStateDatabase interface { - // Store the room state at an event in the database - AddState( - ctx context.Context, - roomNID types.RoomNID, - stateBlockNIDs []types.StateBlockNID, - state []types.StateEntry, - ) (types.StateSnapshotNID, error) - // Look up the state of a room at each event for a list of string event IDs. - // Returns an error if there is an error talking to the database - // Returns a types.MissingEventError if the room state for the event IDs aren't in the database - StateAtEventIDs(ctx context.Context, eventIDs []string) ([]types.StateAtEvent, error) - // Look up the numeric IDs for a list of string event types. - // Returns a map from string event type to numeric ID for the event type. - EventTypeNIDs(ctx context.Context, eventTypes []string) (map[string]types.EventTypeNID, error) - // Look up the numeric IDs for a list of string event state keys. - // Returns a map from string state key to numeric ID for the state key. - EventStateKeyNIDs(ctx context.Context, eventStateKeys []string) (map[string]types.EventStateKeyNID, error) - // Look up the numeric state data IDs for each numeric state snapshot ID - // The returned slice is sorted by numeric state snapshot ID. - StateBlockNIDs(ctx context.Context, stateNIDs []types.StateSnapshotNID) ([]types.StateBlockNIDList, error) - // Look up the state data for each numeric state data ID - // The returned slice is sorted by numeric state data ID. - StateEntries(ctx context.Context, stateBlockNIDs []types.StateBlockNID) ([]types.StateEntryList, error) - // Look up the state data for the state key tuples for each numeric state block ID - // This is used to fetch a subset of the room state at a snapshot. - // If a block doesn't contain any of the requested tuples then it can be discarded from the result. - // The returned slice is sorted by numeric state block ID. - StateEntriesForTuples( - ctx context.Context, - stateBlockNIDs []types.StateBlockNID, - stateKeyTuples []types.StateKeyTuple, - ) ([]types.StateEntryList, error) - // Look up the Events for a list of numeric event IDs. - // Returns a sorted list of events. - Events(ctx context.Context, eventNIDs []types.EventNID) ([]types.Event, error) - // Look up snapshot NID for an event ID string - SnapshotNIDFromEventID(ctx context.Context, eventID string) (types.StateSnapshotNID, error) - // Look up a room version from the room NID. - GetRoomVersionForRoomNID(ctx context.Context, roomNID types.RoomNID) (gomatrixserverlib.RoomVersion, error) -} diff --git a/roomserver/state/shared/shared.go b/roomserver/state/shared/shared.go deleted file mode 100644 index a29b5e403..000000000 --- a/roomserver/state/shared/shared.go +++ /dev/null @@ -1 +0,0 @@ -package shared diff --git a/roomserver/state/state.go b/roomserver/state/state.go index 3f68e0747..9b005ee6a 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -22,7 +22,7 @@ import ( "sort" "time" - "github.com/matrix-org/dendrite/roomserver/state/database" + "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/util" "github.com/prometheus/client_golang/prometheus" @@ -31,10 +31,10 @@ import ( ) type StateResolution struct { - db database.RoomStateDatabase + db storage.Database } -func NewStateResolution(db database.RoomStateDatabase) StateResolution { +func NewStateResolution(db storage.Database) StateResolution { return StateResolution{ db: db, } @@ -86,7 +86,10 @@ func (v StateResolution) LoadStateAtEvent( ) ([]types.StateEntry, error) { snapshotNID, err := v.db.SnapshotNIDFromEventID(ctx, eventID) if err != nil { - return nil, err + return nil, fmt.Errorf("LoadStateAtEvent.SnapshotNIDFromEventID failed for event %s : %s", eventID, err) + } + if snapshotNID == 0 { + return nil, fmt.Errorf("LoadStateAtEvent.SnapshotNIDFromEventID(%s) returned 0 NID, was this event stored?", eventID) } stateEntries, err := v.LoadStateAtSnapshot(ctx, snapshotNID) diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index 50369d806..fb39eca63 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -18,13 +18,51 @@ import ( "context" "github.com/matrix-org/dendrite/roomserver/api" - statedb "github.com/matrix-org/dendrite/roomserver/state/database" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" ) type Database interface { - statedb.RoomStateDatabase + // Store the room state at an event in the database + AddState( + ctx context.Context, + roomNID types.RoomNID, + stateBlockNIDs []types.StateBlockNID, + state []types.StateEntry, + ) (types.StateSnapshotNID, error) + // Look up the state of a room at each event for a list of string event IDs. + // Returns an error if there is an error talking to the database. + // The length of []types.StateAtEvent is guaranteed to equal the length of eventIDs if no error is returned. + // Returns a types.MissingEventError if the room state for the event IDs aren't in the database + StateAtEventIDs(ctx context.Context, eventIDs []string) ([]types.StateAtEvent, error) + // Look up the numeric IDs for a list of string event types. + // Returns a map from string event type to numeric ID for the event type. + EventTypeNIDs(ctx context.Context, eventTypes []string) (map[string]types.EventTypeNID, error) + // Look up the numeric IDs for a list of string event state keys. + // Returns a map from string state key to numeric ID for the state key. + EventStateKeyNIDs(ctx context.Context, eventStateKeys []string) (map[string]types.EventStateKeyNID, error) + // Look up the numeric state data IDs for each numeric state snapshot ID + // The returned slice is sorted by numeric state snapshot ID. + StateBlockNIDs(ctx context.Context, stateNIDs []types.StateSnapshotNID) ([]types.StateBlockNIDList, error) + // Look up the state data for each numeric state data ID + // The returned slice is sorted by numeric state data ID. + StateEntries(ctx context.Context, stateBlockNIDs []types.StateBlockNID) ([]types.StateEntryList, error) + // Look up the state data for the state key tuples for each numeric state block ID + // This is used to fetch a subset of the room state at a snapshot. + // If a block doesn't contain any of the requested tuples then it can be discarded from the result. + // The returned slice is sorted by numeric state block ID. + StateEntriesForTuples( + ctx context.Context, + stateBlockNIDs []types.StateBlockNID, + stateKeyTuples []types.StateKeyTuple, + ) ([]types.StateEntryList, error) + // Look up the Events for a list of numeric event IDs. + // Returns a sorted list of events. + Events(ctx context.Context, eventNIDs []types.EventNID) ([]types.Event, error) + // Look up snapshot NID for an event ID string + SnapshotNIDFromEventID(ctx context.Context, eventID string) (types.StateSnapshotNID, error) + // Look up a room version from the room NID. + GetRoomVersionForRoomNID(ctx context.Context, roomNID types.RoomNID) (gomatrixserverlib.RoomVersion, error) StoreEvent(ctx context.Context, event gomatrixserverlib.Event, txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID) (types.RoomNID, types.StateAtEvent, error) StateEntriesForEventIDs(ctx context.Context, eventIDs []string) ([]types.StateEntry, error) EventStateKeys(ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID) (map[types.EventStateKeyNID]string, error) @@ -34,6 +72,10 @@ type Database interface { GetLatestEventsForUpdate(ctx context.Context, roomNID types.RoomNID) (types.RoomRecentEventsUpdater, error) GetTransactionEventID(ctx context.Context, transactionID string, sessionID int64, userID string) (string, error) RoomNID(ctx context.Context, roomID string) (types.RoomNID, error) + // RoomNIDExcludingStubs is a special variation of RoomNID that will return 0 as if the room + // does not exist if the room has no latest events. This can happen when we've received an + // invite over federation for a room that we don't know anything else about yet. + RoomNIDExcludingStubs(ctx context.Context, roomID string) (types.RoomNID, error) LatestEventIDs(ctx context.Context, roomNID types.RoomNID) ([]gomatrixserverlib.EventReference, types.StateSnapshotNID, int64, error) GetInvitesForUser(ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) (senderUserIDs []types.EventStateKeyNID, err error) SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index 6f2b96610..1d825ecc2 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -20,6 +20,7 @@ import ( "database/sql" "encoding/json" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/internal/sqlutil" // Import the postgres database driver. @@ -36,10 +37,10 @@ type Database struct { } // Open a postgres database. -func Open(dataSourceName string) (*Database, error) { +func Open(dataSourceName string, dbProperties common.DbProperties) (*Database, error) { var d Database var err error - if d.db, err = sqlutil.Open("postgres", dataSourceName); err != nil { + if d.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } if err = d.statements.prepare(d.db); err != nil { @@ -471,6 +472,23 @@ func (d *Database) RoomNID(ctx context.Context, roomID string) (types.RoomNID, e return roomNID, err } +// RoomNIDExcludingStubs implements query.RoomserverQueryAPIDB +func (d *Database) RoomNIDExcludingStubs(ctx context.Context, roomID string) (roomNID types.RoomNID, err error) { + roomNID, err = d.RoomNID(ctx, roomID) + if err != nil { + return + } + latestEvents, _, err := d.statements.selectLatestEventNIDs(ctx, roomNID) + if err != nil { + return + } + if len(latestEvents) == 0 { + roomNID = 0 + return + } + return +} + // LatestEventIDs implements query.RoomserverQueryAPIDatabase func (d *Database) LatestEventIDs( ctx context.Context, roomNID types.RoomNID, diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index d881fa91f..a63596aeb 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -48,11 +48,6 @@ const insertEventSQL = ` ON CONFLICT DO NOTHING; ` -const insertEventResultSQL = ` - SELECT event_nid, state_snapshot_nid FROM roomserver_events - WHERE rowid = last_insert_rowid(); -` - const selectEventSQL = "" + "SELECT event_nid, state_snapshot_nid FROM roomserver_events WHERE event_id = $1" @@ -102,7 +97,6 @@ const selectRoomNIDForEventNIDSQL = "" + type eventStatements struct { db *sql.DB insertEventStmt *sql.Stmt - insertEventResultStmt *sql.Stmt selectEventStmt *sql.Stmt bulkSelectStateEventByIDStmt *sql.Stmt bulkSelectStateAtEventByIDStmt *sql.Stmt @@ -126,7 +120,6 @@ func (s *eventStatements) prepare(db *sql.DB) (err error) { return statementList{ {&s.insertEventStmt, insertEventSQL}, - {&s.insertEventResultStmt, insertEventResultSQL}, {&s.selectEventStmt, selectEventSQL}, {&s.bulkSelectStateEventByIDStmt, bulkSelectStateEventByIDSQL}, {&s.bulkSelectStateAtEventByIDStmt, bulkSelectStateAtEventByIDSQL}, @@ -152,19 +145,22 @@ func (s *eventStatements) insertEvent( referenceSHA256 []byte, authEventNIDs []types.EventNID, depth int64, -) (types.EventNID, types.StateSnapshotNID, error) { - var eventNID int64 - var stateNID int64 - var err error +) (types.EventNID, error) { + // attempt to insert: the last_row_id is the event NID insertStmt := common.TxStmt(txn, s.insertEventStmt) - resultStmt := common.TxStmt(txn, s.insertEventResultStmt) - if _, err = insertStmt.ExecContext( + result, err := insertStmt.ExecContext( ctx, int64(roomNID), int64(eventTypeNID), int64(eventStateKeyNID), eventID, referenceSHA256, eventNIDsAsArray(authEventNIDs), depth, - ); err == nil { - err = resultStmt.QueryRowContext(ctx).Scan(&eventNID, &stateNID) + ) + if err != nil { + return 0, err } - return types.EventNID(eventNID), types.StateSnapshotNID(stateNID), err + modified, err := result.RowsAffected() + if modified == 0 && err == nil { + return 0, sql.ErrNoRows + } + eventNID, err := result.LastInsertId() + return types.EventNID(eventNID), err } func (s *eventStatements) selectEvent( diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index 444a8fdd5..e77fea9cf 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -37,7 +37,7 @@ type Database struct { db *sql.DB } -// Open a postgres database. +// Open a sqlite database. func Open(dataSourceName string) (*Database, error) { var d Database uri, err := url.Parse(dataSourceName) @@ -52,7 +52,7 @@ func Open(dataSourceName string) (*Database, error) { } else { return nil, errors.New("no filename or path in connect string") } - if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs); err != nil { + if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs, nil); err != nil { return nil, err } //d.db.Exec("PRAGMA journal_mode=WAL;") @@ -124,7 +124,7 @@ func (d *Database) StoreEvent( } } - if eventNID, stateNID, err = d.statements.insertEvent( + if eventNID, err = d.statements.insertEvent( ctx, txn, roomNID, @@ -590,6 +590,23 @@ func (d *Database) RoomNID(ctx context.Context, roomID string) (roomNID types.Ro return } +// RoomNIDExcludingStubs implements query.RoomserverQueryAPIDB +func (d *Database) RoomNIDExcludingStubs(ctx context.Context, roomID string) (roomNID types.RoomNID, err error) { + roomNID, err = d.RoomNID(ctx, roomID) + if err != nil { + return + } + latestEvents, _, err := d.statements.selectLatestEventNIDs(ctx, nil, roomNID) + if err != nil { + return + } + if len(latestEvents) == 0 { + roomNID = 0 + return + } + return +} + // LatestEventIDs implements query.RoomserverQueryAPIDatabase func (d *Database) LatestEventIDs( ctx context.Context, roomNID types.RoomNID, diff --git a/roomserver/storage/storage.go b/roomserver/storage/storage.go index 7b9109aa0..99e99a008 100644 --- a/roomserver/storage/storage.go +++ b/roomserver/storage/storage.go @@ -19,22 +19,23 @@ package storage import ( "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/storage/postgres" "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" ) -// NewPublicRoomsServerDatabase opens a database connection. -func Open(dataSourceName string) (Database, error) { +// Open opens a database connection. +func Open(dataSourceName string, dbProperties common.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.Open(dataSourceName) + return postgres.Open(dataSourceName, dbProperties) } switch uri.Scheme { case "postgres": - return postgres.Open(dataSourceName) + return postgres.Open(dataSourceName, dbProperties) case "file": return sqlite3.Open(dataSourceName) default: - return postgres.Open(dataSourceName) + return postgres.Open(dataSourceName, dbProperties) } } diff --git a/roomserver/storage/storage_wasm.go b/roomserver/storage/storage_wasm.go index d7fc352e8..5fa48bc95 100644 --- a/roomserver/storage/storage_wasm.go +++ b/roomserver/storage/storage_wasm.go @@ -18,11 +18,15 @@ import ( "fmt" "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" ) // NewPublicRoomsServerDatabase opens a database connection. -func Open(dataSourceName string) (Database, error) { +func Open( + dataSourceName string, + dbProperties common.DbProperties, // nolint:unparam +) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return nil, fmt.Errorf("Cannot use postgres implementation") diff --git a/roomserver/version/version.go b/roomserver/version/version.go index f2a67e74d..ddd0f23a6 100644 --- a/roomserver/version/version.go +++ b/roomserver/version/version.go @@ -51,15 +51,15 @@ var roomVersions = map[gomatrixserverlib.RoomVersion]RoomVersionDescription{ Stable: true, }, gomatrixserverlib.RoomVersionV5: RoomVersionDescription{ - Supported: false, - Stable: false, + Supported: true, + Stable: true, }, } // DefaultRoomVersion contains the room version that will, by // default, be used to create new rooms on this server. func DefaultRoomVersion() gomatrixserverlib.RoomVersion { - return gomatrixserverlib.RoomVersionV4 + return gomatrixserverlib.RoomVersionV5 } // RoomVersions returns a map of all known room versions to this diff --git a/syncapi/consumers/clientapi.go b/syncapi/consumers/clientapi.go index 17f2c522c..f5b8c43ec 100644 --- a/syncapi/consumers/clientapi.go +++ b/syncapi/consumers/clientapi.go @@ -18,13 +18,13 @@ import ( "context" "encoding/json" + "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/sync" "github.com/matrix-org/dendrite/syncapi/types" log "github.com/sirupsen/logrus" - sarama "gopkg.in/Shopify/sarama.v1" ) // OutputClientDataConsumer consumes events that originated in the client API server. diff --git a/syncapi/consumers/eduserver.go b/syncapi/consumers/eduserver.go index 5491c1e9f..249452af5 100644 --- a/syncapi/consumers/eduserver.go +++ b/syncapi/consumers/eduserver.go @@ -17,6 +17,7 @@ package consumers import ( "encoding/json" + "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/eduserver/api" @@ -24,7 +25,6 @@ import ( "github.com/matrix-org/dendrite/syncapi/sync" "github.com/matrix-org/dendrite/syncapi/types" log "github.com/sirupsen/logrus" - sarama "gopkg.in/Shopify/sarama.v1" ) // OutputTypingEventConsumer consumes events that originated in the EDU server. diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index f1e68c262..987cc5df6 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -19,6 +19,7 @@ import ( "encoding/json" "fmt" + "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/roomserver/api" @@ -27,15 +28,14 @@ import ( "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" - sarama "gopkg.in/Shopify/sarama.v1" ) // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { - roomServerConsumer *common.ContinualConsumer - db storage.Database - notifier *sync.Notifier - query api.RoomserverQueryAPI + rsAPI api.RoomserverInternalAPI + rsConsumer *common.ContinualConsumer + db storage.Database + notifier *sync.Notifier } // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. @@ -44,7 +44,7 @@ func NewOutputRoomEventConsumer( kafkaConsumer sarama.Consumer, n *sync.Notifier, store storage.Database, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, ) *OutputRoomEventConsumer { consumer := common.ContinualConsumer{ @@ -53,10 +53,10 @@ func NewOutputRoomEventConsumer( PartitionStore: store, } s := &OutputRoomEventConsumer{ - roomServerConsumer: &consumer, - db: store, - notifier: n, - query: queryAPI, + rsConsumer: &consumer, + db: store, + notifier: n, + rsAPI: rsAPI, } consumer.ProcessMessage = s.onMessage @@ -65,7 +65,7 @@ func NewOutputRoomEventConsumer( // Start consuming from room servers func (s *OutputRoomEventConsumer) Start() error { - return s.roomServerConsumer.Start() + return s.rsConsumer.Start() } // onMessage is called when the sync server receives a new event from the room server output log. @@ -226,7 +226,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( // from the roomserver using the query API. eventReq := api.QueryEventsByIDRequest{EventIDs: missing} var eventResp api.QueryEventsByIDResponse - if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { + if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { return nil, err } diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 873ee9366..270b0ee95 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -34,7 +34,7 @@ import ( type messagesReq struct { ctx context.Context db storage.Database - queryAPI api.RoomserverQueryAPI + rsAPI api.RoomserverInternalAPI federation *gomatrixserverlib.FederationClient cfg *config.Dendrite roomID string @@ -59,7 +59,7 @@ const defaultMessagesLimit = 10 func OnIncomingMessagesRequest( req *http.Request, db storage.Database, roomID string, federation *gomatrixserverlib.FederationClient, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, cfg *config.Dendrite, ) util.JSONResponse { var err error @@ -135,7 +135,7 @@ func OnIncomingMessagesRequest( mReq := messagesReq{ ctx: req.Context(), db: db, - queryAPI: queryAPI, + rsAPI: rsAPI, federation: federation, cfg: cfg, roomID: roomID, @@ -210,12 +210,16 @@ func (r *messagesReq) retrieveEvents() ( } // Sort the events to ensure we send them in the right order. - events = gomatrixserverlib.HeaderedReverseTopologicalOrdering(events) if r.backwardOrdering { // This reverses the array from old->new to new->old - sort.SliceStable(events, func(i, j int) bool { - return true - }) + reversed := func(in []gomatrixserverlib.HeaderedEvent) []gomatrixserverlib.HeaderedEvent { + out := make([]gomatrixserverlib.HeaderedEvent, len(in)) + for i := 0; i < len(in); i++ { + out[i] = in[len(in)-i-1] + } + return out + } + events = reversed(events) } // Convert all of the events into client events. @@ -226,14 +230,14 @@ func (r *messagesReq) retrieveEvents() ( // change the way topological positions are defined (as depth isn't the most // reliable way to define it), it would be easier and less troublesome to // only have to change it in one place, i.e. the database. - startPos, err := r.db.EventPositionInTopology( + startPos, startStreamPos, err := r.db.EventPositionInTopology( r.ctx, events[0].EventID(), ) if err != nil { err = fmt.Errorf("EventPositionInTopology: for start event %s: %w", events[0].EventID(), err) return } - endPos, err := r.db.EventPositionInTopology( + endPos, endStreamPos, err := r.db.EventPositionInTopology( r.ctx, events[len(events)-1].EventID(), ) if err != nil { @@ -243,10 +247,10 @@ func (r *messagesReq) retrieveEvents() ( // Generate pagination tokens to send to the client using the positions // retrieved previously. start = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, startPos, 0, + types.PaginationTokenTypeTopology, startPos, startStreamPos, ) end = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, endPos, 0, + types.PaginationTokenTypeTopology, endPos, endStreamPos, ) if r.backwardOrdering { @@ -256,6 +260,7 @@ func (r *messagesReq) retrieveEvents() ( // to them by the event on their left, therefore we need to decrement the // end position we send in the response if we're going backward. end.PDUPosition-- + end.EDUTypingPosition += 1000 } // The lowest token value is 1, therefore we need to manually set it to that @@ -282,7 +287,7 @@ func (r *messagesReq) handleEmptyEventsSlice() ( // Check if we have backward extremities for this room. if len(backwardExtremities) > 0 { // If so, retrieve as much events as needed through backfilling. - events, err = r.backfill(backwardExtremities, r.limit) + events, err = r.backfill(r.roomID, backwardExtremities, r.limit) if err != nil { return } @@ -331,7 +336,7 @@ func (r *messagesReq) handleNonEmptyEventsSlice(streamEvents []types.StreamEvent if len(backwardExtremities) > 0 && !isSetLargeEnough && r.backwardOrdering { var pdus []gomatrixserverlib.HeaderedEvent // Only ask the remote server for enough events to reach the limit. - pdus, err = r.backfill(backwardExtremities, r.limit-len(streamEvents)) + pdus, err = r.backfill(r.roomID, backwardExtremities, r.limit-len(streamEvents)) if err != nil { return } @@ -342,10 +347,23 @@ func (r *messagesReq) handleNonEmptyEventsSlice(streamEvents []types.StreamEvent // Append the events ve previously retrieved locally. events = append(events, r.db.StreamEventsToEvents(nil, streamEvents)...) + sort.Sort(eventsByDepth(events)) return } +type eventsByDepth []gomatrixserverlib.HeaderedEvent + +func (e eventsByDepth) Len() int { + return len(e) +} +func (e eventsByDepth) Swap(i, j int) { + e[i], e[j] = e[j], e[i] +} +func (e eventsByDepth) Less(i, j int) bool { + return e[i].Depth() < e[j].Depth() +} + // backfill performs a backfill request over the federation on another // homeserver in the room. // See: https://matrix.org/docs/spec/server_server/latest#get-matrix-federation-v1-backfill-roomid @@ -355,111 +373,46 @@ func (r *messagesReq) handleNonEmptyEventsSlice(streamEvents []types.StreamEvent // event, or if there is no remote homeserver to contact. // Returns an error if there was an issue with retrieving the list of servers in // the room or sending the request. -func (r *messagesReq) backfill(fromEventIDs []string, limit int) ([]gomatrixserverlib.HeaderedEvent, error) { - verReq := api.QueryRoomVersionForRoomRequest{RoomID: r.roomID} - verRes := api.QueryRoomVersionForRoomResponse{} - if err := r.queryAPI.QueryRoomVersionForRoom(r.ctx, &verReq, &verRes); err != nil { - return nil, err - } - - srvToBackfillFrom, err := r.serverToBackfillFrom(fromEventIDs) +func (r *messagesReq) backfill(roomID string, fromEventIDs []string, limit int) ([]gomatrixserverlib.HeaderedEvent, error) { + var res api.QueryBackfillResponse + err := r.rsAPI.QueryBackfill(context.Background(), &api.QueryBackfillRequest{ + RoomID: roomID, + EarliestEventsIDs: fromEventIDs, + Limit: limit, + ServerName: r.cfg.Matrix.ServerName, + }, &res) if err != nil { - return nil, fmt.Errorf("Cannot find server to backfill from: %w", err) + return nil, fmt.Errorf("QueryBackfill failed: %w", err) } + util.GetLogger(r.ctx).WithField("new_events", len(res.Events)).Info("Storing new events from backfill") - headered := make([]gomatrixserverlib.HeaderedEvent, 0) + // TODO: we should only be inserting events into the database from the roomserver's kafka output stream. + // Currently, this can race with live events for the room and cause problems. It's also just a bit unclear + // when you have multiple entry points to write events. - // If the roomserver responded with at least one server that isn't us, - // send it a request for backfill. - util.GetLogger(r.ctx).WithField("server", srvToBackfillFrom).WithField("limit", limit).Info("Backfilling from server") - txn, err := r.federation.Backfill( - r.ctx, srvToBackfillFrom, r.roomID, limit, fromEventIDs, - ) - if err != nil { - return nil, err - } - - for _, p := range txn.PDUs { - event, e := gomatrixserverlib.NewEventFromUntrustedJSON(p, verRes.RoomVersion) - if e != nil { - continue - } - headered = append(headered, event.Headered(verRes.RoomVersion)) - } - util.GetLogger(r.ctx).WithField("server", srvToBackfillFrom).WithField("new_events", len(headered)).Info("Storing new events from backfill") + // we have to order these by depth, starting with the lowest because otherwise the topology tokens + // will skip over events that have the same depth but different stream positions due to the query which is: + // - anything less than the depth OR + // - anything with the same depth and a lower stream position. + sort.Sort(eventsByDepth(res.Events)) // Store the events in the database, while marking them as unfit to show // up in responses to sync requests. - for i := range headered { - if _, err = r.db.WriteEvent( + for i := range res.Events { + _, err = r.db.WriteEvent( r.ctx, - &headered[i], + &res.Events[i], []gomatrixserverlib.HeaderedEvent{}, []string{}, []string{}, nil, true, - ); err != nil { + ) + if err != nil { return nil, err } } - return headered, nil -} - -func (r *messagesReq) serverToBackfillFrom(fromEventIDs []string) (gomatrixserverlib.ServerName, error) { - // Query the list of servers in the room when one of the backward extremities - // was sent. - var serversResponse api.QueryServersInRoomAtEventResponse - serversRequest := api.QueryServersInRoomAtEventRequest{ - RoomID: r.roomID, - EventID: fromEventIDs[0], - } - if err := r.queryAPI.QueryServersInRoomAtEvent(r.ctx, &serversRequest, &serversResponse); err != nil { - util.GetLogger(r.ctx).WithError(err).Warn("Failed to query servers in room at event, falling back to event sender") - // FIXME: We shouldn't be doing this but in situations where we have already backfilled once - // the query API doesn't work as backfilled events do not make it to the room server. - // This means QueryServersInRoomAtEvent returns an error as it doesn't have the event ID in question. - // We need to inject backfilled events into the room server and store them appropriately. - events, err := r.db.Events(r.ctx, fromEventIDs) - if err != nil { - return "", err - } - if len(events) == 0 { - // should be impossible as these event IDs are backwards extremities - return "", fmt.Errorf("backfill: missing backwards extremities, event IDs: %s", fromEventIDs) - } - // The rationale here is that the last event was unlikely to be sent by us, so poke the server who sent it. - // We shouldn't be doing this really, but as a heuristic it should work pretty well for now. - for _, e := range events { - _, srv, srverr := gomatrixserverlib.SplitID('@', e.Sender()) - if srverr != nil { - util.GetLogger(r.ctx).WithError(srverr).Warn("Failed to extract domain from event sender") - continue - } - if srv != r.cfg.Matrix.ServerName { - return srv, nil - } - } - // no valid events which have a remote server, fail. - return "", err - } - - // Use the first server from the response, except if that server is us. - // In that case, use the second one if the roomserver responded with - // enough servers. If not, use an empty string to prevent the backfill - // from happening as there's no server to direct the request towards. - // TODO: Be smarter at selecting the server to direct the request - // towards. - srvToBackfillFrom := serversResponse.Servers[0] - if srvToBackfillFrom == r.cfg.Matrix.ServerName { - if len(serversResponse.Servers) > 1 { - srvToBackfillFrom = serversResponse.Servers[1] - } else { - util.GetLogger(r.ctx).Info("Not enough servers to backfill from") - return "", nil - } - } - return srvToBackfillFrom, nil + return res.Events, nil } // setToDefault returns the default value for the "to" query parameter of a @@ -476,13 +429,13 @@ func setToDefault( // go 1 earlier than the first event so we correctly fetch the earliest event to = types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) } else { - var pos types.StreamPosition - pos, err = db.MaxTopologicalPosition(ctx, roomID) + var pos, stream types.StreamPosition + pos, stream, err = db.MaxTopologicalPosition(ctx, roomID) if err != nil { return } - to = types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, pos, 0) + to = types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, pos, stream) } return diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index 9078b87ff..5a36a279f 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -40,7 +40,7 @@ const pathPrefixR0 = "/_matrix/client/r0" func Setup( apiMux *mux.Router, srp *sync.RequestPool, syncDB storage.Database, deviceDB devices.Database, federation *gomatrixserverlib.FederationClient, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, cfg *config.Dendrite, ) { r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() @@ -61,6 +61,6 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], federation, queryAPI, cfg) + return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], federation, rsAPI, cfg) })).Methods(http.MethodGet, http.MethodOptions) } diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index a3efd8d58..7d6376438 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -28,26 +28,83 @@ import ( type Database interface { common.PartitionStorer + // AllJoinedUsersInRooms returns a map of room ID to a list of all joined user IDs. AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) + // Events lookups a list of event by their event ID. + // Returns a list of events matching the requested IDs found in the database. + // If an event is not found in the database then it will be omitted from the list. + // Returns an error if there was a problem talking with the database. + // Does not include any transaction IDs in the returned events. Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error) - WriteEvent(context.Context, *gomatrixserverlib.HeaderedEvent, []gomatrixserverlib.HeaderedEvent, []string, []string, *api.TransactionID, bool) (types.StreamPosition, error) + // WriteEvent into the database. It is not safe to call this function from multiple goroutines, as it would create races + // when generating the sync stream position for this event. Returns the sync stream position for the inserted event. + // Returns an error if there was a problem inserting this event. + WriteEvent(ctx context.Context, ev *gomatrixserverlib.HeaderedEvent, addStateEvents []gomatrixserverlib.HeaderedEvent, + addStateEventIDs []string, removeStateEventIDs []string, transactionID *api.TransactionID, excludeFromSync bool) (types.StreamPosition, error) + // GetStateEvent returns the Matrix state event of a given type for a given room with a given state key + // If no event could be found, returns nil + // If there was an issue during the retrieval, returns an error GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error) + // GetStateEventsForRoom fetches the state events for a given room. + // Returns an empty slice if no state events could be found for this room. + // Returns an error if there was an issue with the retrieval. GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter) (stateEvents []gomatrixserverlib.HeaderedEvent, err error) + // SyncPosition returns the latest positions for syncing. SyncPosition(ctx context.Context) (types.PaginationToken, error) + // IncrementalSync returns all the data needed in order to create an incremental + // sync response for the given user. Events returned will include any client + // transaction IDs associated with the given device. These transaction IDs come + // from when the device sent the event via an API that included a transaction + // ID. IncrementalSync(ctx context.Context, device authtypes.Device, fromPos, toPos types.PaginationToken, numRecentEventsPerRoom int, wantFullState bool) (*types.Response, error) + // CompleteSync returns a complete /sync API response for the given user. CompleteSync(ctx context.Context, userID string, numRecentEventsPerRoom int) (*types.Response, error) + // GetAccountDataInRange returns all account data for a given user inserted or + // updated between two given positions + // Returns a map following the format data[roomID] = []dataTypes + // If no data is retrieved, returns an empty map + // If there was an issue with the retrieval, returns an error GetAccountDataInRange(ctx context.Context, userID string, oldPos, newPos types.StreamPosition, accountDataFilterPart *gomatrixserverlib.EventFilter) (map[string][]string, error) + // UpsertAccountData keeps track of new or updated account data, by saving the type + // of the new/updated data, and the user ID and room ID the data is related to (empty) + // room ID means the data isn't specific to any room) + // If no data with the given type, user ID and room ID exists in the database, + // creates a new row, else update the existing one + // Returns an error if there was an issue with the upsert UpsertAccountData(ctx context.Context, userID, roomID, dataType string) (types.StreamPosition, error) + // AddInviteEvent stores a new invite event for a user. + // If the invite was successfully stored this returns the stream ID it was stored at. + // Returns an error if there was a problem communicating with the database. AddInviteEvent(ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent) (types.StreamPosition, error) + // RetireInviteEvent removes an old invite event from the database. + // Returns an error if there was a problem communicating with the database. RetireInviteEvent(ctx context.Context, inviteEventID string) error + // SetTypingTimeoutCallback sets a callback function that is called right after + // a user is removed from the typing user list due to timeout. SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) + // AddTypingUser adds a typing user to the typing cache. + // Returns the newly calculated sync position for typing notifications. AddTypingUser(userID, roomID string, expireTime *time.Time) types.StreamPosition + // RemoveTypingUser removes a typing user from the typing cache. + // Returns the newly calculated sync position for typing notifications. RemoveTypingUser(userID, roomID string) types.StreamPosition + // GetEventsInRange retrieves all of the events on a given ordering using the + // given extremities and limit. GetEventsInRange(ctx context.Context, from, to *types.PaginationToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error) - EventPositionInTopology(ctx context.Context, eventID string) (types.StreamPosition, error) + // EventPositionInTopology returns the depth and stream position of the given event. + EventPositionInTopology(ctx context.Context, eventID string) (depth types.StreamPosition, stream types.StreamPosition, err error) + // EventsAtTopologicalPosition returns all of the events matching a given + // position in the topology of a given room. EventsAtTopologicalPosition(ctx context.Context, roomID string, pos types.StreamPosition) ([]types.StreamEvent, error) + // BackwardExtremitiesForRoom returns the event IDs of all of the backward + // extremities we know of for a given room. BackwardExtremitiesForRoom(ctx context.Context, roomID string) (backwardExtremities []string, err error) - MaxTopologicalPosition(ctx context.Context, roomID string) (types.StreamPosition, error) + // MaxTopologicalPosition returns the highest topological position for a given room. + MaxTopologicalPosition(ctx context.Context, roomID string) (depth types.StreamPosition, stream types.StreamPosition, err error) + // StreamEventsToEvents converts streamEvent to Event. If device is non-nil and + // matches the streamevent.transactionID device then the transaction ID gets + // added to the unsigned section of the output event. StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent + // SyncStreamPosition returns the latest position in the sync stream. Returns 0 if there are no events yet. SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) } diff --git a/syncapi/storage/postgres/backward_extremities_table.go b/syncapi/storage/postgres/backward_extremities_table.go deleted file mode 100644 index cb3629644..000000000 --- a/syncapi/storage/postgres/backward_extremities_table.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2018 New Vector Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package postgres - -import ( - "context" - "database/sql" - - "github.com/matrix-org/dendrite/common" -) - -// The purpose of this table is to keep track of backwards extremities for a room. -// Backwards extremities are the earliest (DAG-wise) known events which we have -// the entire event JSON. These event IDs are used in federation requests to fetch -// even earlier events. -// -// We persist the previous event IDs as well, one per row, so when we do fetch even -// earlier events we can simply delete rows which referenced it. Consider the graph: -// A -// | Event C has 1 prev_event ID: A. -// B C -// |___| Event D has 2 prev_event IDs: B and C. -// | -// D -// The earliest known event we have is D, so this table has 2 rows. -// A backfill request gives us C but not B. We delete rows where prev_event=C. This -// still means that D is a backwards extremity as we do not have event B. However, event -// C is *also* a backwards extremity at this point as we do not have event A. Later, -// when we fetch event B, we delete rows where prev_event=B. This then removes D as -// a backwards extremity because there are no more rows with event_id=B. -const backwardExtremitiesSchema = ` --- Stores output room events received from the roomserver. -CREATE TABLE IF NOT EXISTS syncapi_backward_extremities ( - -- The 'room_id' key for the event. - room_id TEXT NOT NULL, - -- The event ID for the last known event. This is the backwards extremity. - event_id TEXT NOT NULL, - -- The prev_events for the last known event. This is used to update extremities. - prev_event_id TEXT NOT NULL, - - PRIMARY KEY(room_id, event_id, prev_event_id) -); -` - -const insertBackwardExtremitySQL = "" + - "INSERT INTO syncapi_backward_extremities (room_id, event_id, prev_event_id)" + - " VALUES ($1, $2, $3)" + - " ON CONFLICT DO NOTHING" - -const selectBackwardExtremitiesForRoomSQL = "" + - "SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1" - -const deleteBackwardExtremitySQL = "" + - "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" - -type backwardExtremitiesStatements struct { - insertBackwardExtremityStmt *sql.Stmt - selectBackwardExtremitiesForRoomStmt *sql.Stmt - deleteBackwardExtremityStmt *sql.Stmt -} - -func (s *backwardExtremitiesStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(backwardExtremitiesSchema) - if err != nil { - return - } - if s.insertBackwardExtremityStmt, err = db.Prepare(insertBackwardExtremitySQL); err != nil { - return - } - if s.selectBackwardExtremitiesForRoomStmt, err = db.Prepare(selectBackwardExtremitiesForRoomSQL); err != nil { - return - } - if s.deleteBackwardExtremityStmt, err = db.Prepare(deleteBackwardExtremitySQL); err != nil { - return - } - return -} - -func (s *backwardExtremitiesStatements) insertsBackwardExtremity( - ctx context.Context, txn *sql.Tx, roomID, eventID string, prevEventID string, -) (err error) { - _, err = txn.Stmt(s.insertBackwardExtremityStmt).ExecContext(ctx, roomID, eventID, prevEventID) - return -} - -func (s *backwardExtremitiesStatements) selectBackwardExtremitiesForRoom( - ctx context.Context, roomID string, -) (eventIDs []string, err error) { - rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID) - if err != nil { - return - } - defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") - - for rows.Next() { - var eID string - if err = rows.Scan(&eID); err != nil { - return - } - - eventIDs = append(eventIDs, eID) - } - - return eventIDs, rows.Err() -} - -func (s *backwardExtremitiesStatements) deleteBackwardExtremity( - ctx context.Context, txn *sql.Tx, roomID, knownEventID string, -) (err error) { - _, err = txn.Stmt(s.deleteBackwardExtremityStmt).ExecContext(ctx, roomID, knownEventID) - return -} diff --git a/syncapi/storage/postgres/output_room_events_topology_table.go b/syncapi/storage/postgres/output_room_events_topology_table.go index 280d4ec39..51cbd50d0 100644 --- a/syncapi/storage/postgres/output_room_events_topology_table.go +++ b/syncapi/storage/postgres/output_room_events_topology_table.go @@ -32,35 +32,44 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events_topology ( -- The place of the event in the room's topology. This can usually be determined -- from the event's depth. topological_position BIGINT NOT NULL, + stream_position BIGINT NOT NULL, -- The 'room_id' key for the event. room_id TEXT NOT NULL ); -- The topological order will be used in events selection and ordering -CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_topological_position_idx ON syncapi_output_room_events_topology(topological_position, room_id); +CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_topological_position_idx ON syncapi_output_room_events_topology(topological_position, stream_position, room_id); ` const insertEventInTopologySQL = "" + - "INSERT INTO syncapi_output_room_events_topology (event_id, topological_position, room_id)" + - " VALUES ($1, $2, $3)" + - " ON CONFLICT (topological_position, room_id) DO UPDATE SET event_id = $1" + "INSERT INTO syncapi_output_room_events_topology (event_id, topological_position, room_id, stream_position)" + + " VALUES ($1, $2, $3, $4)" + + " ON CONFLICT (topological_position, stream_position, room_id) DO UPDATE SET event_id = $1" const selectEventIDsInRangeASCSQL = "" + "SELECT event_id FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1 AND topological_position > $2 AND topological_position <= $3" + - " ORDER BY topological_position ASC LIMIT $4" + " WHERE room_id = $1 AND" + + "(topological_position > $2 AND topological_position < $3) OR" + + "(topological_position = $4 AND stream_position <= $5)" + + " ORDER BY topological_position ASC, stream_position ASC LIMIT $6" const selectEventIDsInRangeDESCSQL = "" + "SELECT event_id FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1 AND topological_position > $2 AND topological_position <= $3" + - " ORDER BY topological_position DESC LIMIT $4" + " WHERE room_id = $1 AND" + + "(topological_position > $2 AND topological_position < $3) OR" + + "(topological_position = $4 AND stream_position <= $5)" + + " ORDER BY topological_position DESC, stream_position DESC LIMIT $6" const selectPositionInTopologySQL = "" + - "SELECT topological_position FROM syncapi_output_room_events_topology" + + "SELECT topological_position, stream_position FROM syncapi_output_room_events_topology" + " WHERE event_id = $1" + // Select the max topological position for the room, then sort by stream position and take the highest, + // returning both topological and stream positions. const selectMaxPositionInTopologySQL = "" + - "SELECT MAX(topological_position) FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1" + "SELECT topological_position, stream_position FROM syncapi_output_room_events_topology" + + " WHERE topological_position=(" + + "SELECT MAX(topological_position) FROM syncapi_output_room_events_topology WHERE room_id=$1" + + ") ORDER BY stream_position DESC LIMIT 1" const selectEventIDsFromPositionSQL = "" + "SELECT event_id FROM syncapi_output_room_events_topology" + @@ -104,10 +113,10 @@ func (s *outputRoomEventsTopologyStatements) prepare(db *sql.DB) (err error) { // insertEventInTopology inserts the given event in the room's topology, based // on the event's depth. func (s *outputRoomEventsTopologyStatements) insertEventInTopology( - ctx context.Context, event *gomatrixserverlib.HeaderedEvent, + ctx context.Context, event *gomatrixserverlib.HeaderedEvent, pos types.StreamPosition, ) (err error) { _, err = s.insertEventInTopologyStmt.ExecContext( - ctx, event.EventID(), event.Depth(), event.RoomID(), + ctx, event.EventID(), event.Depth(), event.RoomID(), pos, ) return } @@ -116,7 +125,7 @@ func (s *outputRoomEventsTopologyStatements) insertEventInTopology( // given range in a given room's topological order. // Returns an empty slice if no events match the given range. func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( - ctx context.Context, roomID string, fromPos, toPos types.StreamPosition, + ctx context.Context, roomID string, fromPos, toPos, toMicroPos types.StreamPosition, limit int, chronologicalOrder bool, ) (eventIDs []string, err error) { // Decide on the selection's order according to whether chronological order @@ -129,7 +138,7 @@ func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( } // Query the event IDs. - rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, limit) + rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, toPos, toMicroPos, limit) if err == sql.ErrNoRows { // If no event matched the request, return an empty slice. return []string{}, nil @@ -154,15 +163,15 @@ func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( // topology of the room it belongs to. func (s *outputRoomEventsTopologyStatements) selectPositionInTopology( ctx context.Context, eventID string, -) (pos types.StreamPosition, err error) { - err = s.selectPositionInTopologyStmt.QueryRowContext(ctx, eventID).Scan(&pos) +) (pos, spos types.StreamPosition, err error) { + err = s.selectPositionInTopologyStmt.QueryRowContext(ctx, eventID).Scan(&pos, &spos) return } func (s *outputRoomEventsTopologyStatements) selectMaxPositionInTopology( ctx context.Context, roomID string, -) (pos types.StreamPosition, err error) { - err = s.selectMaxPositionInTopologyStmt.QueryRowContext(ctx, roomID).Scan(&pos) +) (pos types.StreamPosition, spos types.StreamPosition, err error) { + err = s.selectMaxPositionInTopologyStmt.QueryRowContext(ctx, roomID).Scan(&pos, &spos) return } diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index 7fd75f066..1845ac386 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -32,6 +32,7 @@ import ( _ "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -56,14 +57,14 @@ type SyncServerDatasource struct { invites inviteEventsStatements eduCache *cache.EDUCache topology outputRoomEventsTopologyStatements - backwardExtremities backwardExtremitiesStatements + backwardExtremities tables.BackwardsExtremities } // NewSyncServerDatasource creates a new sync server database -func NewSyncServerDatasource(dbDataSourceName string) (*SyncServerDatasource, error) { +func NewSyncServerDatasource(dbDataSourceName string, dbProperties common.DbProperties) (*SyncServerDatasource, error) { var d SyncServerDatasource var err error - if d.db, err = sqlutil.Open("postgres", dbDataSourceName); err != nil { + if d.db, err = sqlutil.Open("postgres", dbDataSourceName, dbProperties); err != nil { return nil, err } if err = d.PartitionOffsetStatements.Prepare(d.db, "syncapi"); err != nil { @@ -75,32 +76,27 @@ func NewSyncServerDatasource(dbDataSourceName string) (*SyncServerDatasource, er if err = d.events.prepare(d.db); err != nil { return nil, err } - if err := d.roomstate.prepare(d.db); err != nil { + if err = d.roomstate.prepare(d.db); err != nil { return nil, err } - if err := d.invites.prepare(d.db); err != nil { + if err = d.invites.prepare(d.db); err != nil { return nil, err } - if err := d.topology.prepare(d.db); err != nil { + if err = d.topology.prepare(d.db); err != nil { return nil, err } - if err := d.backwardExtremities.prepare(d.db); err != nil { + d.backwardExtremities, err = tables.NewBackwardsExtremities(d.db, &tables.PostgresBackwardsExtremitiesStatements{}) + if err != nil { return nil, err } d.eduCache = cache.New() return &d, nil } -// AllJoinedUsersInRooms returns a map of room ID to a list of all joined user IDs. func (d *SyncServerDatasource) AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) { return d.roomstate.selectJoinedUsers(ctx) } -// Events lookups a list of event by their event ID. -// Returns a list of events matching the requested IDs found in the database. -// If an event is not found in the database then it will be omitted from the list. -// Returns an error if there was a problem talking with the database. -// Does not include any transaction IDs in the returned events. func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error) { streamEvents, err := d.events.selectEvents(ctx, nil, eventIDs) if err != nil { @@ -116,7 +112,7 @@ func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([ // the events listed in the event's 'prev_events'. This function also updates the backwards extremities table // to account for the fact that the given event is no longer a backwards extremity, but may be marked as such. func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *gomatrixserverlib.HeaderedEvent) error { - if err := d.backwardExtremities.deleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { + if err := d.backwardExtremities.DeleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { return err } @@ -137,7 +133,7 @@ func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, tx // If the event is missing, consider it a backward extremity. if !found { - if err = d.backwardExtremities.insertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil { + if err = d.backwardExtremities.InsertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil { return err } } @@ -146,9 +142,6 @@ func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, tx return nil } -// WriteEvent into the database. It is not safe to call this function from multiple goroutines, as it would create races -// when generating the sync stream position for this event. Returns the sync stream position for the inserted event. -// Returns an error if there was a problem inserting this event. func (d *SyncServerDatasource) WriteEvent( ctx context.Context, ev *gomatrixserverlib.HeaderedEvent, @@ -166,7 +159,7 @@ func (d *SyncServerDatasource) WriteEvent( } pduPosition = pos - if err = d.topology.insertEventInTopology(ctx, ev); err != nil { + if err = d.topology.insertEventInTopology(ctx, ev, pos); err != nil { return err } @@ -219,18 +212,12 @@ func (d *SyncServerDatasource) updateRoomState( return nil } -// GetStateEvent returns the Matrix state event of a given type for a given room with a given state key -// If no event could be found, returns nil -// If there was an issue during the retrieval, returns an error func (d *SyncServerDatasource) GetStateEvent( ctx context.Context, roomID, evType, stateKey string, ) (*gomatrixserverlib.HeaderedEvent, error) { return d.roomstate.selectStateEvent(ctx, roomID, evType, stateKey) } -// GetStateEventsForRoom fetches the state events for a given room. -// Returns an empty slice if no state events could be found for this room. -// Returns an error if there was an issue with the retrieval. func (d *SyncServerDatasource) GetStateEventsForRoom( ctx context.Context, roomID string, stateFilter *gomatrixserverlib.StateFilter, ) (stateEvents []gomatrixserverlib.HeaderedEvent, err error) { @@ -241,8 +228,6 @@ func (d *SyncServerDatasource) GetStateEventsForRoom( return } -// GetEventsInRange retrieves all of the events on a given ordering using the -// given extremities and limit. func (d *SyncServerDatasource) GetEventsInRange( ctx context.Context, from, to *types.PaginationToken, @@ -255,12 +240,13 @@ func (d *SyncServerDatasource) GetEventsInRange( if from.Type == types.PaginationTokenTypeTopology { // Determine the backward and forward limit, i.e. the upper and lower // limits to the selection in the room's topology, from the direction. - var backwardLimit, forwardLimit types.StreamPosition + var backwardLimit, forwardLimit, forwardMicroLimit types.StreamPosition if backwardOrdering { // Backward ordering is antichronological (latest event to oldest // one). backwardLimit = to.PDUPosition forwardLimit = from.PDUPosition + forwardMicroLimit = from.EDUTypingPosition } else { // Forward ordering is chronological (oldest event to latest one). backwardLimit = from.PDUPosition @@ -270,7 +256,7 @@ func (d *SyncServerDatasource) GetEventsInRange( // Select the event IDs from the defined range. var eIDs []string eIDs, err = d.topology.selectEventIDsInRange( - ctx, roomID, backwardLimit, forwardLimit, limit, !backwardOrdering, + ctx, roomID, backwardLimit, forwardLimit, forwardMicroLimit, limit, !backwardOrdering, ) if err != nil { return @@ -304,29 +290,22 @@ func (d *SyncServerDatasource) GetEventsInRange( return } -// SyncPosition returns the latest positions for syncing. func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (types.PaginationToken, error) { return d.syncPositionTx(ctx, nil) } -// BackwardExtremitiesForRoom returns the event IDs of all of the backward -// extremities we know of for a given room. func (d *SyncServerDatasource) BackwardExtremitiesForRoom( ctx context.Context, roomID string, ) (backwardExtremities []string, err error) { - return d.backwardExtremities.selectBackwardExtremitiesForRoom(ctx, roomID) + return d.backwardExtremities.SelectBackwardExtremitiesForRoom(ctx, roomID) } -// MaxTopologicalPosition returns the highest topological position for a given -// room. func (d *SyncServerDatasource) MaxTopologicalPosition( ctx context.Context, roomID string, -) (types.StreamPosition, error) { +) (depth types.StreamPosition, stream types.StreamPosition, err error) { return d.topology.selectMaxPositionInTopology(ctx, roomID) } -// EventsAtTopologicalPosition returns all of the events matching a given -// position in the topology of a given room. func (d *SyncServerDatasource) EventsAtTopologicalPosition( ctx context.Context, roomID string, pos types.StreamPosition, ) ([]types.StreamEvent, error) { @@ -340,11 +319,10 @@ func (d *SyncServerDatasource) EventsAtTopologicalPosition( func (d *SyncServerDatasource) EventPositionInTopology( ctx context.Context, eventID string, -) (types.StreamPosition, error) { +) (depth types.StreamPosition, stream types.StreamPosition, err error) { return d.topology.selectPositionInTopology(ctx, eventID) } -// SyncStreamPosition returns the latest position in the sync stream. Returns 0 if there are no events yet. func (d *SyncServerDatasource) SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) { return d.syncStreamPositionTx(ctx, nil) } @@ -509,11 +487,6 @@ func (d *SyncServerDatasource) addEDUDeltaToResponse( return } -// IncrementalSync returns all the data needed in order to create an incremental -// sync response for the given user. Events returned will include any client -// transaction IDs associated with the given device. These transaction IDs come -// from when the device sent the event via an API that included a transaction -// ID. func (d *SyncServerDatasource) IncrementalSync( ctx context.Context, device authtypes.Device, @@ -613,8 +586,8 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( // Retrieve the backward topology position, i.e. the position of the // oldest event in the room's topology. - var backwardTopologyPos types.StreamPosition - backwardTopologyPos, err = d.topology.selectPositionInTopology(ctx, recentStreamEvents[0].EventID()) + var backwardTopologyPos, backwardStreamPos types.StreamPosition + backwardTopologyPos, backwardStreamPos, err = d.topology.selectPositionInTopology(ctx, recentStreamEvents[0].EventID()) if backwardTopologyPos-1 <= 0 { backwardTopologyPos = types.StreamPosition(1) } else { @@ -627,7 +600,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( stateEvents = removeDuplicates(stateEvents, recentEvents) jr := types.NewJoinResponse() jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, backwardTopologyPos, 0, + types.PaginationTokenTypeTopology, backwardTopologyPos, backwardStreamPos, ).String() jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = true @@ -643,7 +616,6 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( return res, toPos, joinedRoomIDs, err } -// CompleteSync returns a complete /sync API response for the given user. func (d *SyncServerDatasource) CompleteSync( ctx context.Context, userID string, numRecentEventsPerRoom int, ) (*types.Response, error) { @@ -675,11 +647,6 @@ var txReadOnlySnapshot = sql.TxOptions{ ReadOnly: true, } -// GetAccountDataInRange returns all account data for a given user inserted or -// updated between two given positions -// Returns a map following the format data[roomID] = []dataTypes -// If no data is retrieved, returns an empty map -// If there was an issue with the retrieval, returns an error func (d *SyncServerDatasource) GetAccountDataInRange( ctx context.Context, userID string, oldPos, newPos types.StreamPosition, accountDataFilterPart *gomatrixserverlib.EventFilter, @@ -687,29 +654,18 @@ func (d *SyncServerDatasource) GetAccountDataInRange( return d.accountData.selectAccountDataInRange(ctx, userID, oldPos, newPos, accountDataFilterPart) } -// UpsertAccountData keeps track of new or updated account data, by saving the type -// of the new/updated data, and the user ID and room ID the data is related to (empty) -// room ID means the data isn't specific to any room) -// If no data with the given type, user ID and room ID exists in the database, -// creates a new row, else update the existing one -// Returns an error if there was an issue with the upsert func (d *SyncServerDatasource) UpsertAccountData( ctx context.Context, userID, roomID, dataType string, ) (types.StreamPosition, error) { return d.accountData.insertAccountData(ctx, userID, roomID, dataType) } -// AddInviteEvent stores a new invite event for a user. -// If the invite was successfully stored this returns the stream ID it was stored at. -// Returns an error if there was a problem communicating with the database. func (d *SyncServerDatasource) AddInviteEvent( ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, ) (types.StreamPosition, error) { return d.invites.insertInviteEvent(ctx, inviteEvent) } -// RetireInviteEvent removes an old invite event from the database. -// Returns an error if there was a problem communicating with the database. func (d *SyncServerDatasource) RetireInviteEvent( ctx context.Context, inviteEventID string, ) error { @@ -723,16 +679,12 @@ func (d *SyncServerDatasource) SetTypingTimeoutCallback(fn cache.TimeoutCallback d.eduCache.SetTimeoutCallback(fn) } -// AddTypingUser adds a typing user to the typing cache. -// Returns the newly calculated sync position for typing notifications. func (d *SyncServerDatasource) AddTypingUser( userID, roomID string, expireTime *time.Time, ) types.StreamPosition { return types.StreamPosition(d.eduCache.AddTypingUser(userID, roomID, expireTime)) } -// RemoveTypingUser removes a typing user from the typing cache. -// Returns the newly calculated sync position for typing notifications. func (d *SyncServerDatasource) RemoveTypingUser( userID, roomID string, ) types.StreamPosition { @@ -752,11 +704,7 @@ func (d *SyncServerDatasource) addInvitesToResponse( return err } for roomID, inviteEvent := range invites { - ir := types.NewInviteResponse() - ir.InviteState.Events = gomatrixserverlib.ToClientEvents( - []gomatrixserverlib.Event{inviteEvent.Event}, gomatrixserverlib.FormatSync, - ) - // TODO: add the invite state from the invite event. + ir := types.NewInviteResponse(inviteEvent) res.Rooms.Invite[roomID] = *ir } return nil @@ -767,9 +715,9 @@ func (d *SyncServerDatasource) addInvitesToResponse( func (d *SyncServerDatasource) getBackwardTopologyPos( ctx context.Context, events []types.StreamEvent, -) (pos types.StreamPosition) { +) (pos, spos types.StreamPosition) { if len(events) > 0 { - pos, _ = d.topology.selectPositionInTopology(ctx, events[0].EventID()) + pos, spos, _ = d.topology.selectPositionInTopology(ctx, events[0].EventID()) } if pos-1 <= 0 { pos = types.StreamPosition(1) @@ -808,14 +756,14 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( } recentEvents := d.StreamEventsToEvents(device, recentStreamEvents) delta.stateEvents = removeDuplicates(delta.stateEvents, recentEvents) // roll back - backwardTopologyPos := d.getBackwardTopologyPos(ctx, recentStreamEvents) + backwardTopologyPos, backwardStreamPos := d.getBackwardTopologyPos(ctx, recentStreamEvents) switch delta.membership { case gomatrixserverlib.Join: jr := types.NewJoinResponse() jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, backwardTopologyPos, 0, + types.PaginationTokenTypeTopology, backwardTopologyPos, backwardStreamPos, ).String() jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true @@ -828,7 +776,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( // no longer in the room. lr := types.NewLeaveResponse() lr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, backwardTopologyPos, 0, + types.PaginationTokenTypeTopology, backwardTopologyPos, backwardStreamPos, ).String() lr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true @@ -1075,9 +1023,6 @@ func (d *SyncServerDatasource) currentStateStreamEventsForRoom( return s, nil } -// StreamEventsToEvents converts streamEvent to Event. If device is non-nil and -// matches the streamevent.transactionID device then the transaction ID gets -// added to the unsigned section of the output event. func (d *SyncServerDatasource) StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent { out := make([]gomatrixserverlib.HeaderedEvent, len(in)) for i := 0; i < len(in); i++ { diff --git a/syncapi/storage/sqlite3/backward_extremities_table.go b/syncapi/storage/sqlite3/backward_extremities_table.go deleted file mode 100644 index 3d8cb91fc..000000000 --- a/syncapi/storage/sqlite3/backward_extremities_table.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2018 New Vector Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sqlite3 - -import ( - "context" - "database/sql" - - "github.com/matrix-org/dendrite/common" -) - -// The purpose of this table is to keep track of backwards extremities for a room. -// Backwards extremities are the earliest (DAG-wise) known events which we have -// the entire event JSON. These event IDs are used in federation requests to fetch -// even earlier events. -// -// We persist the previous event IDs as well, one per row, so when we do fetch even -// earlier events we can simply delete rows which referenced it. Consider the graph: -// A -// | Event C has 1 prev_event ID: A. -// B C -// |___| Event D has 2 prev_event IDs: B and C. -// | -// D -// The earliest known event we have is D, so this table has 2 rows. -// A backfill request gives us C but not B. We delete rows where prev_event=C. This -// still means that D is a backwards extremity as we do not have event B. However, event -// C is *also* a backwards extremity at this point as we do not have event A. Later, -// when we fetch event B, we delete rows where prev_event=B. This then removes D as -// a backwards extremity because there are no more rows with event_id=B. -const backwardExtremitiesSchema = ` --- Stores output room events received from the roomserver. -CREATE TABLE IF NOT EXISTS syncapi_backward_extremities ( - -- The 'room_id' key for the event. - room_id TEXT NOT NULL, - -- The event ID for the last known event. This is the backwards extremity. - event_id TEXT NOT NULL, - -- The prev_events for the last known event. This is used to update extremities. - prev_event_id TEXT NOT NULL, - - PRIMARY KEY(room_id, event_id, prev_event_id) -); -` - -const insertBackwardExtremitySQL = "" + - "INSERT INTO syncapi_backward_extremities (room_id, event_id, prev_event_id)" + - " VALUES ($1, $2, $3)" + - " ON CONFLICT (room_id, event_id, prev_event_id) DO NOTHING" - -const selectBackwardExtremitiesForRoomSQL = "" + - "SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1" - -const deleteBackwardExtremitySQL = "" + - "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" - -type backwardExtremitiesStatements struct { - insertBackwardExtremityStmt *sql.Stmt - selectBackwardExtremitiesForRoomStmt *sql.Stmt - deleteBackwardExtremityStmt *sql.Stmt -} - -func (s *backwardExtremitiesStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(backwardExtremitiesSchema) - if err != nil { - return - } - if s.insertBackwardExtremityStmt, err = db.Prepare(insertBackwardExtremitySQL); err != nil { - return - } - if s.selectBackwardExtremitiesForRoomStmt, err = db.Prepare(selectBackwardExtremitiesForRoomSQL); err != nil { - return - } - if s.deleteBackwardExtremityStmt, err = db.Prepare(deleteBackwardExtremitySQL); err != nil { - return - } - return -} - -func (s *backwardExtremitiesStatements) insertsBackwardExtremity( - ctx context.Context, txn *sql.Tx, roomID, eventID string, prevEventID string, -) (err error) { - _, err = txn.Stmt(s.insertBackwardExtremityStmt).ExecContext(ctx, roomID, eventID, prevEventID) - return -} - -func (s *backwardExtremitiesStatements) selectBackwardExtremitiesForRoom( - ctx context.Context, roomID string, -) (eventIDs []string, err error) { - rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID) - if err != nil { - return - } - defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") - - for rows.Next() { - var eID string - if err = rows.Scan(&eID); err != nil { - return - } - - eventIDs = append(eventIDs, eID) - } - - return eventIDs, rows.Err() -} - -func (s *backwardExtremitiesStatements) deleteBackwardExtremity( - ctx context.Context, txn *sql.Tx, roomID, knownEventID string, -) (err error) { - _, err = txn.Stmt(s.deleteBackwardExtremityStmt).ExecContext(ctx, roomID, knownEventID) - return -} diff --git a/syncapi/storage/sqlite3/output_room_events_topology_table.go b/syncapi/storage/sqlite3/output_room_events_topology_table.go index a2944c2f9..0d313d7c6 100644 --- a/syncapi/storage/sqlite3/output_room_events_topology_table.go +++ b/syncapi/storage/sqlite3/output_room_events_topology_table.go @@ -27,37 +27,42 @@ const outputRoomEventsTopologySchema = ` -- Stores output room events received from the roomserver. CREATE TABLE IF NOT EXISTS syncapi_output_room_events_topology ( event_id TEXT PRIMARY KEY, - topological_position BIGINT NOT NULL, + topological_position BIGINT NOT NULL, + stream_position BIGINT NOT NULL, room_id TEXT NOT NULL, - UNIQUE(topological_position, room_id) + UNIQUE(topological_position, room_id, stream_position) ); -- The topological order will be used in events selection and ordering --- CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_topological_position_idx ON syncapi_output_room_events_topology(topological_position, room_id); +-- CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_topological_position_idx ON syncapi_output_room_events_topology(topological_position, stream_position, room_id); ` const insertEventInTopologySQL = "" + - "INSERT INTO syncapi_output_room_events_topology (event_id, topological_position, room_id)" + - " VALUES ($1, $2, $3)" + - " ON CONFLICT (topological_position, room_id) DO UPDATE SET event_id = $1" + "INSERT INTO syncapi_output_room_events_topology (event_id, topological_position, room_id, stream_position)" + + " VALUES ($1, $2, $3, $4)" + + " ON CONFLICT DO NOTHING" const selectEventIDsInRangeASCSQL = "" + "SELECT event_id FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1 AND topological_position > $2 AND topological_position <= $3" + - " ORDER BY topological_position ASC LIMIT $4" + " WHERE room_id = $1 AND" + + "(topological_position > $2 AND topological_position < $3) OR" + + "(topological_position = $4 AND stream_position <= $5)" + + " ORDER BY topological_position ASC, stream_position ASC LIMIT $6" const selectEventIDsInRangeDESCSQL = "" + - "SELECT event_id FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1 AND topological_position > $2 AND topological_position <= $3" + - " ORDER BY topological_position DESC LIMIT $4" + "SELECT event_id FROM syncapi_output_room_events_topology" + + " WHERE room_id = $1 AND" + + "(topological_position > $2 AND topological_position < $3) OR" + + "(topological_position = $4 AND stream_position <= $5)" + + " ORDER BY topological_position DESC, stream_position DESC LIMIT $6" const selectPositionInTopologySQL = "" + - "SELECT topological_position FROM syncapi_output_room_events_topology" + + "SELECT topological_position, stream_position FROM syncapi_output_room_events_topology" + " WHERE event_id = $1" const selectMaxPositionInTopologySQL = "" + - "SELECT MAX(topological_position) FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1" + "SELECT MAX(topological_position), stream_position FROM syncapi_output_room_events_topology" + + " WHERE room_id = $1 ORDER BY stream_position DESC" const selectEventIDsFromPositionSQL = "" + "SELECT event_id FROM syncapi_output_room_events_topology" + @@ -101,11 +106,11 @@ func (s *outputRoomEventsTopologyStatements) prepare(db *sql.DB) (err error) { // insertEventInTopology inserts the given event in the room's topology, based // on the event's depth. func (s *outputRoomEventsTopologyStatements) insertEventInTopology( - ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, + ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, pos types.StreamPosition, ) (err error) { stmt := common.TxStmt(txn, s.insertEventInTopologyStmt) _, err = stmt.ExecContext( - ctx, event.EventID(), event.Depth(), event.RoomID(), + ctx, event.EventID(), event.Depth(), event.RoomID(), pos, ) return } @@ -115,7 +120,7 @@ func (s *outputRoomEventsTopologyStatements) insertEventInTopology( // Returns an empty slice if no events match the given range. func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( ctx context.Context, txn *sql.Tx, roomID string, - fromPos, toPos types.StreamPosition, + fromPos, toPos, toMicroPos types.StreamPosition, limit int, chronologicalOrder bool, ) (eventIDs []string, err error) { // Decide on the selection's order according to whether chronological order @@ -128,7 +133,7 @@ func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( } // Query the event IDs. - rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, limit) + rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, toPos, toMicroPos, limit) if err == sql.ErrNoRows { // If no event matched the request, return an empty slice. return []string{}, nil @@ -152,17 +157,17 @@ func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( // topology of the room it belongs to. func (s *outputRoomEventsTopologyStatements) selectPositionInTopology( ctx context.Context, txn *sql.Tx, eventID string, -) (pos types.StreamPosition, err error) { +) (pos types.StreamPosition, spos types.StreamPosition, err error) { stmt := common.TxStmt(txn, s.selectPositionInTopologyStmt) - err = stmt.QueryRowContext(ctx, eventID).Scan(&pos) + err = stmt.QueryRowContext(ctx, eventID).Scan(&pos, &spos) return } func (s *outputRoomEventsTopologyStatements) selectMaxPositionInTopology( ctx context.Context, txn *sql.Tx, roomID string, -) (pos types.StreamPosition, err error) { +) (pos types.StreamPosition, spos types.StreamPosition, err error) { stmt := common.TxStmt(txn, s.selectMaxPositionInTopologyStmt) - err = stmt.QueryRowContext(ctx, roomID).Scan(&pos) + err = stmt.QueryRowContext(ctx, roomID).Scan(&pos, &spos) return } diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 29051cd06..314ea2aa3 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -35,6 +35,7 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -60,7 +61,7 @@ type SyncServerDatasource struct { invites inviteEventsStatements eduCache *cache.EDUCache topology outputRoomEventsTopologyStatements - backwardExtremities backwardExtremitiesStatements + backwardExtremities tables.BackwardsExtremities } // NewSyncServerDatasource creates a new sync server database @@ -79,7 +80,7 @@ func NewSyncServerDatasource(dataSourceName string) (*SyncServerDatasource, erro } else { return nil, errors.New("no filename or path in connect string") } - if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs); err != nil { + if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs, nil); err != nil { return nil, err } if err = d.prepare(); err != nil { @@ -102,16 +103,17 @@ func (d *SyncServerDatasource) prepare() (err error) { if err = d.events.prepare(d.db, &d.streamID); err != nil { return err } - if err := d.roomstate.prepare(d.db, &d.streamID); err != nil { + if err = d.roomstate.prepare(d.db, &d.streamID); err != nil { return err } - if err := d.invites.prepare(d.db, &d.streamID); err != nil { + if err = d.invites.prepare(d.db, &d.streamID); err != nil { return err } - if err := d.topology.prepare(d.db); err != nil { + if err = d.topology.prepare(d.db); err != nil { return err } - if err := d.backwardExtremities.prepare(d.db); err != nil { + d.backwardExtremities, err = tables.NewBackwardsExtremities(d.db, &tables.SqliteBackwardsExtremitiesStatements{}) + if err != nil { return err } return nil @@ -142,7 +144,7 @@ func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([ // the events listed in the event's 'prev_events'. This function also updates the backwards extremities table // to account for the fact that the given event is no longer a backwards extremity, but may be marked as such. func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *gomatrixserverlib.HeaderedEvent) error { - if err := d.backwardExtremities.deleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { + if err := d.backwardExtremities.DeleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { return err } @@ -163,7 +165,7 @@ func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, tx // If the event is missing, consider it a backward extremity. if !found { - if err = d.backwardExtremities.insertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil { + if err = d.backwardExtremities.InsertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil { return err } } @@ -192,7 +194,7 @@ func (d *SyncServerDatasource) WriteEvent( } pduPosition = pos - if err = d.topology.insertEventInTopology(ctx, txn, ev); err != nil { + if err = d.topology.insertEventInTopology(ctx, txn, ev, pos); err != nil { return err } @@ -279,14 +281,16 @@ func (d *SyncServerDatasource) GetEventsInRange( // events must be retrieved from the rooms' topology table rather than the // table contaning the syncapi server's whole stream of events. if from.Type == types.PaginationTokenTypeTopology { + // TODO: ARGH CONFUSING // Determine the backward and forward limit, i.e. the upper and lower // limits to the selection in the room's topology, from the direction. - var backwardLimit, forwardLimit types.StreamPosition + var backwardLimit, forwardLimit, forwardMicroLimit types.StreamPosition if backwardOrdering { // Backward ordering is antichronological (latest event to oldest // one). backwardLimit = to.PDUPosition forwardLimit = from.PDUPosition + forwardMicroLimit = from.EDUTypingPosition } else { // Forward ordering is chronological (oldest event to latest one). backwardLimit = from.PDUPosition @@ -296,7 +300,7 @@ func (d *SyncServerDatasource) GetEventsInRange( // Select the event IDs from the defined range. var eIDs []string eIDs, err = d.topology.selectEventIDsInRange( - ctx, nil, roomID, backwardLimit, forwardLimit, limit, !backwardOrdering, + ctx, nil, roomID, backwardLimit, forwardLimit, forwardMicroLimit, limit, !backwardOrdering, ) if err != nil { return @@ -326,8 +330,7 @@ func (d *SyncServerDatasource) GetEventsInRange( return } } - - return + return events, err } // SyncPosition returns the latest positions for syncing. @@ -344,14 +347,14 @@ func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (tok types.Pagi func (d *SyncServerDatasource) BackwardExtremitiesForRoom( ctx context.Context, roomID string, ) (backwardExtremities []string, err error) { - return d.backwardExtremities.selectBackwardExtremitiesForRoom(ctx, roomID) + return d.backwardExtremities.SelectBackwardExtremitiesForRoom(ctx, roomID) } // MaxTopologicalPosition returns the highest topological position for a given // room. func (d *SyncServerDatasource) MaxTopologicalPosition( ctx context.Context, roomID string, -) (types.StreamPosition, error) { +) (types.StreamPosition, types.StreamPosition, error) { return d.topology.selectMaxPositionInTopology(ctx, nil, roomID) } @@ -370,7 +373,7 @@ func (d *SyncServerDatasource) EventsAtTopologicalPosition( func (d *SyncServerDatasource) EventPositionInTopology( ctx context.Context, eventID string, -) (types.StreamPosition, error) { +) (depth types.StreamPosition, stream types.StreamPosition, err error) { return d.topology.selectPositionInTopology(ctx, nil, eventID) } @@ -431,6 +434,7 @@ func (d *SyncServerDatasource) syncPositionTx( } sp.PDUPosition = types.StreamPosition(maxEventID) sp.EDUTypingPosition = types.StreamPosition(d.eduCache.GetLatestSyncPosition()) + sp.Type = types.PaginationTokenTypeStream return } @@ -649,12 +653,13 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( // Retrieve the backward topology position, i.e. the position of the // oldest event in the room's topology. - var backwardTopologyPos types.StreamPosition - backwardTopologyPos, err = d.topology.selectPositionInTopology(ctx, txn, recentStreamEvents[0].EventID()) + var backwardTopologyPos, backwardTopologyStreamPos types.StreamPosition + backwardTopologyPos, backwardTopologyStreamPos, err = d.topology.selectPositionInTopology(ctx, txn, recentStreamEvents[0].EventID()) if backwardTopologyPos-1 <= 0 { backwardTopologyPos = types.StreamPosition(1) } else { backwardTopologyPos-- + backwardTopologyStreamPos += 1000 // this has to be bigger than the number of events we backfill per request } // We don't include a device here as we don't need to send down @@ -663,7 +668,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( stateEvents = removeDuplicates(stateEvents, recentEvents) jr := types.NewJoinResponse() jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, backwardTopologyPos, 0, + types.PaginationTokenTypeTopology, backwardTopologyPos, backwardTopologyStreamPos, ).String() jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = true @@ -799,11 +804,7 @@ func (d *SyncServerDatasource) addInvitesToResponse( return err } for roomID, inviteEvent := range invites { - ir := types.NewInviteResponse() - ir.InviteState.Events = gomatrixserverlib.HeaderedToClientEvents( - []gomatrixserverlib.HeaderedEvent{inviteEvent}, gomatrixserverlib.FormatSync, - ) - // TODO: add the invite state from the invite event. + ir := types.NewInviteResponse(inviteEvent) res.Rooms.Invite[roomID] = *ir } return nil @@ -814,14 +815,17 @@ func (d *SyncServerDatasource) addInvitesToResponse( func (d *SyncServerDatasource) getBackwardTopologyPos( ctx context.Context, txn *sql.Tx, events []types.StreamEvent, -) (pos types.StreamPosition) { +) (pos, spos types.StreamPosition) { if len(events) > 0 { - pos, _ = d.topology.selectPositionInTopology(ctx, txn, events[0].EventID()) + pos, spos, _ = d.topology.selectPositionInTopology(ctx, txn, events[0].EventID()) } + // go to the previous position so we don't pull out the same event twice + // FIXME: This could be done more nicely by being explicit with inclusive/exclusive rules if pos-1 <= 0 { pos = types.StreamPosition(1) } else { pos = pos - 1 + spos += 1000 // this has to be bigger than the number of events we backfill per request } return } @@ -855,14 +859,14 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( } recentEvents := d.StreamEventsToEvents(device, recentStreamEvents) delta.stateEvents = removeDuplicates(delta.stateEvents, recentEvents) - backwardTopologyPos := d.getBackwardTopologyPos(ctx, txn, recentStreamEvents) + backwardTopologyPos, backwardStreamPos := d.getBackwardTopologyPos(ctx, txn, recentStreamEvents) switch delta.membership { case gomatrixserverlib.Join: jr := types.NewJoinResponse() jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, backwardTopologyPos, 0, + types.PaginationTokenTypeTopology, backwardTopologyPos, backwardStreamPos, ).String() jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true @@ -875,7 +879,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( // no longer in the room. lr := types.NewLeaveResponse() lr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, backwardTopologyPos, 0, + types.PaginationTokenTypeTopology, backwardTopologyPos, backwardStreamPos, ).String() lr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true diff --git a/syncapi/storage/storage.go b/syncapi/storage/storage.go index c56db0635..9574f37bd 100644 --- a/syncapi/storage/storage.go +++ b/syncapi/storage/storage.go @@ -19,22 +19,23 @@ package storage import ( "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/syncapi/storage/postgres" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" ) -// NewPublicRoomsServerDatabase opens a database connection. -func NewSyncServerDatasource(dataSourceName string) (Database, error) { +// NewSyncServerDatasource opens a database connection. +func NewSyncServerDatasource(dataSourceName string, dbProperties common.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.NewSyncServerDatasource(dataSourceName) + return postgres.NewSyncServerDatasource(dataSourceName, dbProperties) } switch uri.Scheme { case "postgres": - return postgres.NewSyncServerDatasource(dataSourceName) + return postgres.NewSyncServerDatasource(dataSourceName, dbProperties) case "file": return sqlite3.NewSyncServerDatasource(dataSourceName) default: - return postgres.NewSyncServerDatasource(dataSourceName) + return postgres.NewSyncServerDatasource(dataSourceName, dbProperties) } } diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go new file mode 100644 index 000000000..b951efa45 --- /dev/null +++ b/syncapi/storage/storage_test.go @@ -0,0 +1,530 @@ +package storage_test + +import ( + "context" + "crypto/ed25519" + "fmt" + "testing" + "time" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/gomatrixserverlib" +) + +var ( + ctx = context.Background() + emptyStateKey = "" + testOrigin = gomatrixserverlib.ServerName("hollow.knight") + testRoomID = fmt.Sprintf("!hallownest:%s", testOrigin) + testUserIDA = fmt.Sprintf("@hornet:%s", testOrigin) + testUserIDB = fmt.Sprintf("@paleking:%s", testOrigin) + testUserDeviceA = authtypes.Device{ + UserID: testUserIDA, + ID: "device_id_A", + DisplayName: "Device A", + } + testRoomVersion = gomatrixserverlib.RoomVersionV4 + testKeyID = gomatrixserverlib.KeyID("ed25519:storage_test") + testPrivateKey = ed25519.NewKeyFromSeed([]byte{ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + }) +) + +func MustCreateEvent(t *testing.T, roomID string, prevs []gomatrixserverlib.HeaderedEvent, b *gomatrixserverlib.EventBuilder) gomatrixserverlib.HeaderedEvent { + b.RoomID = roomID + if prevs != nil { + prevIDs := make([]string, len(prevs)) + for i := range prevs { + prevIDs[i] = prevs[i].EventID() + } + b.PrevEvents = prevIDs + } + e, err := b.Build(time.Now(), testOrigin, testKeyID, testPrivateKey, testRoomVersion) + if err != nil { + t.Fatalf("failed to build event: %s", err) + } + return e.Headered(testRoomVersion) +} + +func MustCreateDatabase(t *testing.T) storage.Database { + db, err := sqlite3.NewSyncServerDatasource("file::memory:") + if err != nil { + t.Fatalf("NewSyncServerDatasource returned %s", err) + } + return db +} + +// Create a list of events which include a create event, join event and some messages. +func SimpleRoom(t *testing.T, roomID, userA, userB string) (msgs []gomatrixserverlib.HeaderedEvent, state []gomatrixserverlib.HeaderedEvent) { + var events []gomatrixserverlib.HeaderedEvent + events = append(events, MustCreateEvent(t, roomID, nil, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"room_version":"4","creator":"%s"}`, userA)), + Type: "m.room.create", + StateKey: &emptyStateKey, + Sender: userA, + Depth: int64(len(events) + 1), + })) + state = append(state, events[len(events)-1]) + events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"membership":"join"}`)), + Type: "m.room.member", + StateKey: &userA, + Sender: userA, + Depth: int64(len(events) + 1), + })) + state = append(state, events[len(events)-1]) + for i := 0; i < 10; i++ { + events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"body":"Message A %d"}`, i+1)), + Type: "m.room.message", + Sender: userA, + Depth: int64(len(events) + 1), + })) + } + events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"membership":"join"}`)), + Type: "m.room.member", + StateKey: &userB, + Sender: userB, + Depth: int64(len(events) + 1), + })) + state = append(state, events[len(events)-1]) + for i := 0; i < 10; i++ { + events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"body":"Message B %d"}`, i+1)), + Type: "m.room.message", + Sender: userB, + Depth: int64(len(events) + 1), + })) + } + + return events, state +} + +func MustWriteEvents(t *testing.T, db storage.Database, events []gomatrixserverlib.HeaderedEvent) (positions []types.StreamPosition) { + for _, ev := range events { + var addStateEvents []gomatrixserverlib.HeaderedEvent + var addStateEventIDs []string + var removeStateEventIDs []string + if ev.StateKey() != nil { + addStateEvents = append(addStateEvents, ev) + addStateEventIDs = append(addStateEventIDs, ev.EventID()) + } + pos, err := db.WriteEvent(ctx, &ev, addStateEvents, addStateEventIDs, removeStateEventIDs, nil, false) + if err != nil { + t.Fatalf("WriteEvent failed: %s", err) + } + fmt.Println("Event ID", ev.EventID(), " spos=", pos, "depth=", ev.Depth()) + positions = append(positions, pos) + } + return +} + +func TestWriteEvents(t *testing.T) { + t.Parallel() + db := MustCreateDatabase(t) + events, _ := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) + MustWriteEvents(t, db, events) +} + +// These tests assert basic functionality of the IncrementalSync and CompleteSync functions. +func TestSyncResponse(t *testing.T) { + t.Parallel() + db := MustCreateDatabase(t) + events, state := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) + positions := MustWriteEvents(t, db, events) + latest, err := db.SyncPosition(ctx) + if err != nil { + t.Fatalf("failed to get SyncPosition: %s", err) + } + + testCases := []struct { + Name string + DoSync func() (*types.Response, error) + WantTimeline []gomatrixserverlib.HeaderedEvent + WantState []gomatrixserverlib.HeaderedEvent + }{ + // The purpose of this test is to make sure that incremental syncs are including up to the latest events. + // It's a basic sanity test that sync works. It creates a `since` token that is on the penultimate event. + // It makes sure the response includes the final event. + { + Name: "IncrementalSync penultimate", + DoSync: func() (*types.Response, error) { + from := types.NewPaginationTokenFromTypeAndPosition( // pretend we are at the penultimate event + types.PaginationTokenTypeStream, positions[len(positions)-2], types.StreamPosition(0), + ) + return db.IncrementalSync(ctx, testUserDeviceA, *from, latest, 5, false) + }, + WantTimeline: events[len(events)-1:], + }, + // The purpose of this test is to check that passing a `numRecentEventsPerRoom` correctly limits the + // number of returned events. This is critical for big rooms hence the test here. + { + Name: "IncrementalSync limited", + DoSync: func() (*types.Response, error) { + from := types.NewPaginationTokenFromTypeAndPosition( // pretend we are 10 events behind + types.PaginationTokenTypeStream, positions[len(positions)-11], types.StreamPosition(0), + ) + // limit is set to 5 + return db.IncrementalSync(ctx, testUserDeviceA, *from, latest, 5, false) + }, + // want the last 5 events, NOT the last 10. + WantTimeline: events[len(events)-5:], + }, + // The purpose of this test is to check that CompleteSync returns all the current state as well as + // honouring the `numRecentEventsPerRoom` value + { + Name: "CompleteSync limited", + DoSync: func() (*types.Response, error) { + // limit set to 5 + return db.CompleteSync(ctx, testUserIDA, 5) + }, + // want the last 5 events + WantTimeline: events[len(events)-5:], + // want all state for the room + WantState: state, + }, + // The purpose of this test is to check that CompleteSync can return everything with a high enough + // `numRecentEventsPerRoom`. + { + Name: "CompleteSync", + DoSync: func() (*types.Response, error) { + return db.CompleteSync(ctx, testUserIDA, len(events)+1) + }, + WantTimeline: events, + // We want no state at all as that field in /sync is the delta between the token (beginning of time) + // and the START of the timeline. + }, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(st *testing.T) { + res, err := tc.DoSync() + if err != nil { + st.Fatalf("failed to do sync: %s", err) + } + next := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeStream, latest.PDUPosition, latest.EDUTypingPosition) + if res.NextBatch != next.String() { + st.Errorf("NextBatch got %s want %s", res.NextBatch, next.String()) + } + roomRes, ok := res.Rooms.Join[testRoomID] + if !ok { + st.Fatalf("IncrementalSync response missing room %s - response: %+v", testRoomID, res) + } + assertEventsEqual(st, "state for "+testRoomID, false, roomRes.State.Events, tc.WantState) + assertEventsEqual(st, "timeline for "+testRoomID, false, roomRes.Timeline.Events, tc.WantTimeline) + }) + } +} + +func TestGetEventsInRangeWithPrevBatch(t *testing.T) { + t.Parallel() + db := MustCreateDatabase(t) + events, _ := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) + positions := MustWriteEvents(t, db, events) + latest, err := db.SyncPosition(ctx) + if err != nil { + t.Fatalf("failed to get SyncPosition: %s", err) + } + from := types.NewPaginationTokenFromTypeAndPosition( + types.PaginationTokenTypeStream, positions[len(positions)-2], types.StreamPosition(0), + ) + + res, err := db.IncrementalSync(ctx, testUserDeviceA, *from, latest, 5, false) + if err != nil { + t.Fatalf("failed to IncrementalSync with latest token") + } + roomRes, ok := res.Rooms.Join[testRoomID] + if !ok { + t.Fatalf("IncrementalSync response missing room %s - response: %+v", testRoomID, res) + } + // returns the last event "Message 10" + assertEventsEqual(t, "IncrementalSync Timeline", false, roomRes.Timeline.Events, reversed(events[len(events)-1:])) + + prev := roomRes.Timeline.PrevBatch + if prev == "" { + t.Fatalf("IncrementalSync expected prev_batch token") + } + prevBatchToken, err := types.NewPaginationTokenFromString(prev) + if err != nil { + t.Fatalf("failed to NewPaginationTokenFromString : %s", err) + } + // backpaginate 5 messages starting at the latest position. + // head towards the beginning of time + to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) + paginatedEvents, err := db.GetEventsInRange(ctx, prevBatchToken, to, testRoomID, 5, true) + if err != nil { + t.Fatalf("GetEventsInRange returned an error: %s", err) + } + gots := gomatrixserverlib.HeaderedToClientEvents(db.StreamEventsToEvents(&testUserDeviceA, paginatedEvents), gomatrixserverlib.FormatAll) + assertEventsEqual(t, "", true, gots, reversed(events[len(events)-6:len(events)-1])) +} + +// The purpose of this test is to ensure that backfill does indeed go backwards, using a stream token. +func TestGetEventsInRangeWithStreamToken(t *testing.T) { + t.Parallel() + db := MustCreateDatabase(t) + events, _ := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) + MustWriteEvents(t, db, events) + latest, err := db.SyncPosition(ctx) + if err != nil { + t.Fatalf("failed to get SyncPosition: %s", err) + } + // head towards the beginning of time + to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) + + // backpaginate 5 messages starting at the latest position. + paginatedEvents, err := db.GetEventsInRange(ctx, &latest, to, testRoomID, 5, true) + if err != nil { + t.Fatalf("GetEventsInRange returned an error: %s", err) + } + gots := gomatrixserverlib.HeaderedToClientEvents(db.StreamEventsToEvents(&testUserDeviceA, paginatedEvents), gomatrixserverlib.FormatAll) + assertEventsEqual(t, "", true, gots, reversed(events[len(events)-5:])) +} + +// The purpose of this test is to ensure that backfill does indeed go backwards, using a topology token +func TestGetEventsInRangeWithTopologyToken(t *testing.T) { + t.Parallel() + db := MustCreateDatabase(t) + events, _ := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) + MustWriteEvents(t, db, events) + latest, latestStream, err := db.MaxTopologicalPosition(ctx, testRoomID) + if err != nil { + t.Fatalf("failed to get MaxTopologicalPosition: %s", err) + } + from := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, latest, latestStream) + // head towards the beginning of time + to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) + + // backpaginate 5 messages starting at the latest position. + paginatedEvents, err := db.GetEventsInRange(ctx, from, to, testRoomID, 5, true) + if err != nil { + t.Fatalf("GetEventsInRange returned an error: %s", err) + } + gots := gomatrixserverlib.HeaderedToClientEvents(db.StreamEventsToEvents(&testUserDeviceA, paginatedEvents), gomatrixserverlib.FormatAll) + assertEventsEqual(t, "", true, gots, reversed(events[len(events)-5:])) +} + +// The purpose of this test is to make sure that backpagination returns all events, even if some events have the same depth. +// For cases where events have the same depth, the streaming token should be used to tie break so events written via WriteEvent +// will appear FIRST when going backwards. This test creates a DAG like: +// .-----> Message ---. +// Create -> Membership --------> Message -------> Message +// `-----> Message ---` +// depth 1 2 3 4 +// +// With a total depth of 4. It tests that: +// - Backpagination over the whole fork should include all messages and not leave any out. +// - Backpagination from the middle of the fork should not return duplicates (things later than the token). +func TestGetEventsInRangeWithEventsSameDepth(t *testing.T) { + t.Parallel() + db := MustCreateDatabase(t) + + var events []gomatrixserverlib.HeaderedEvent + events = append(events, MustCreateEvent(t, testRoomID, nil, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"room_version":"4","creator":"%s"}`, testUserIDA)), + Type: "m.room.create", + StateKey: &emptyStateKey, + Sender: testUserIDA, + Depth: int64(len(events) + 1), + })) + events = append(events, MustCreateEvent(t, testRoomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"membership":"join"}`)), + Type: "m.room.member", + StateKey: &testUserIDA, + Sender: testUserIDA, + Depth: int64(len(events) + 1), + })) + // fork the dag into three, same prev_events and depth + parent := []gomatrixserverlib.HeaderedEvent{events[len(events)-1]} + depth := int64(len(events) + 1) + for i := 0; i < 3; i++ { + events = append(events, MustCreateEvent(t, testRoomID, parent, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"body":"Message A %d"}`, i+1)), + Type: "m.room.message", + Sender: testUserIDA, + Depth: depth, + })) + } + // merge the fork, prev_events are all 3 messages, depth is increased by 1. + events = append(events, MustCreateEvent(t, testRoomID, events[len(events)-3:], &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"body":"Message merge"}`)), + Type: "m.room.message", + Sender: testUserIDA, + Depth: depth + 1, + })) + MustWriteEvents(t, db, events) + latestPos, latestStreamPos, err := db.EventPositionInTopology(ctx, events[len(events)-1].EventID()) + if err != nil { + t.Fatalf("failed to get EventPositionInTopology: %s", err) + } + topoPos, streamPos, err := db.EventPositionInTopology(ctx, events[len(events)-3].EventID()) // Message 2 + if err != nil { + t.Fatalf("failed to get EventPositionInTopology for event: %s", err) + } + fromLatest := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, latestPos, latestStreamPos) + fromFork := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, topoPos, streamPos) + // head towards the beginning of time + to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) + + testCases := []struct { + Name string + From *types.PaginationToken + Limit int + Wants []gomatrixserverlib.HeaderedEvent + }{ + { + Name: "Pagination over the whole fork", + From: fromLatest, + Limit: 5, + Wants: reversed(events[len(events)-5:]), + }, + { + Name: "Paginating to the middle of the fork", + From: fromLatest, + Limit: 2, + Wants: reversed(events[len(events)-2:]), + }, + { + Name: "Pagination FROM the middle of the fork", + From: fromFork, + Limit: 3, + Wants: reversed(events[len(events)-5 : len(events)-2]), + }, + } + + for _, tc := range testCases { + // backpaginate messages starting at the latest position. + paginatedEvents, err := db.GetEventsInRange(ctx, tc.From, to, testRoomID, tc.Limit, true) + if err != nil { + t.Fatalf("%s GetEventsInRange returned an error: %s", tc.Name, err) + } + gots := gomatrixserverlib.HeaderedToClientEvents(db.StreamEventsToEvents(&testUserDeviceA, paginatedEvents), gomatrixserverlib.FormatAll) + assertEventsEqual(t, tc.Name, true, gots, tc.Wants) + } +} + +// The purpose of this test is to make sure that events are returned in the right *order* when they have been inserted in a manner similar to +// how any kind of backfill operation will insert the events. This test inserts the SimpleRoom events in a manner similar to how backfill over +// federation would: +// - First inserts join event of test user C +// - Inserts chunks of history in strata e.g (25-30, 20-25, 15-20, 10-15, 5-10, 0-5). +// The test then does a backfill to ensure that the response is ordered correctly according to depth. +func TestGetEventsInRangeWithEventsInsertedLikeBackfill(t *testing.T) { + t.Parallel() + db := MustCreateDatabase(t) + events, _ := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) + + // "federation" join + userC := fmt.Sprintf("@radiance:%s", testOrigin) + joinEvent := MustCreateEvent(t, testRoomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"membership":"join"}`)), + Type: "m.room.member", + StateKey: &userC, + Sender: userC, + Depth: int64(len(events) + 1), + }) + MustWriteEvents(t, db, []gomatrixserverlib.HeaderedEvent{joinEvent}) + + // Sync will return this for the prev_batch + from := topologyTokenBefore(t, db, joinEvent.EventID()) + + // inject events in batches as if they were from backfill + // e.g [1,2,3,4,5,6] => [4,5,6] , [1,2,3] + chunkSize := 5 + for i := len(events); i >= 0; i -= chunkSize { + start := i - chunkSize + if start < 0 { + start = 0 + } + backfill := events[start:i] + MustWriteEvents(t, db, backfill) + } + + // head towards the beginning of time + to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) + + // starting at `from`, backpaginate to the beginning of time, asserting as we go. + chunkSize = 3 + events = reversed(events) + for i := 0; i < len(events); i += chunkSize { + paginatedEvents, err := db.GetEventsInRange(ctx, from, to, testRoomID, chunkSize, true) + if err != nil { + t.Fatalf("GetEventsInRange returned an error: %s", err) + } + gots := gomatrixserverlib.HeaderedToClientEvents(db.StreamEventsToEvents(&testUserDeviceA, paginatedEvents), gomatrixserverlib.FormatAll) + endi := i + chunkSize + if endi > len(events) { + endi = len(events) + } + assertEventsEqual(t, from.String(), true, gots, events[i:endi]) + from = topologyTokenBefore(t, db, paginatedEvents[len(paginatedEvents)-1].EventID()) + } +} + +func assertEventsEqual(t *testing.T, msg string, checkRoomID bool, gots []gomatrixserverlib.ClientEvent, wants []gomatrixserverlib.HeaderedEvent) { + if len(gots) != len(wants) { + t.Fatalf("%s response returned %d events, want %d", msg, len(gots), len(wants)) + } + for i := range gots { + g := gots[i] + w := wants[i] + if g.EventID != w.EventID() { + t.Errorf("%s event[%d] event_id mismatch: got %s want %s", msg, i, g.EventID, w.EventID()) + } + if g.Sender != w.Sender() { + t.Errorf("%s event[%d] sender mismatch: got %s want %s", msg, i, g.Sender, w.Sender()) + } + if checkRoomID && g.RoomID != w.RoomID() { + t.Errorf("%s event[%d] room_id mismatch: got %s want %s", msg, i, g.RoomID, w.RoomID()) + } + if g.Type != w.Type() { + t.Errorf("%s event[%d] event type mismatch: got %s want %s", msg, i, g.Type, w.Type()) + } + if g.OriginServerTS != w.OriginServerTS() { + t.Errorf("%s event[%d] origin_server_ts mismatch: got %v want %v", msg, i, g.OriginServerTS, w.OriginServerTS()) + } + if string(g.Content) != string(w.Content()) { + t.Errorf("%s event[%d] content mismatch: got %s want %s", msg, i, string(g.Content), string(w.Content())) + } + if string(g.Unsigned) != string(w.Unsigned()) { + t.Errorf("%s event[%d] unsigned mismatch: got %s want %s", msg, i, string(g.Unsigned), string(w.Unsigned())) + } + if (g.StateKey == nil && w.StateKey() != nil) || (g.StateKey != nil && w.StateKey() == nil) { + t.Errorf("%s event[%d] state_key [not] missing: got %v want %v", msg, i, g.StateKey, w.StateKey()) + continue + } + if g.StateKey != nil { + if !w.StateKeyEquals(*g.StateKey) { + t.Errorf("%s event[%d] state_key mismatch: got %s want %s", msg, i, *g.StateKey, *w.StateKey()) + } + } + } +} + +func topologyTokenBefore(t *testing.T, db storage.Database, eventID string) *types.PaginationToken { + pos, spos, err := db.EventPositionInTopology(ctx, eventID) + if err != nil { + t.Fatalf("failed to get EventPositionInTopology: %s", err) + } + + if pos-1 <= 0 { + pos = types.StreamPosition(1) + } else { + pos = pos - 1 + spos += 1000 // this has to be bigger than the chunk limit + } + return types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, pos, spos) +} + +func reversed(in []gomatrixserverlib.HeaderedEvent) []gomatrixserverlib.HeaderedEvent { + out := make([]gomatrixserverlib.HeaderedEvent, len(in)) + for i := 0; i < len(in); i++ { + out[i] = in[len(in)-i-1] + } + return out +} diff --git a/syncapi/storage/storage_wasm.go b/syncapi/storage/storage_wasm.go index 43806a012..84bd9df96 100644 --- a/syncapi/storage/storage_wasm.go +++ b/syncapi/storage/storage_wasm.go @@ -18,11 +18,15 @@ import ( "fmt" "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" ) // NewPublicRoomsServerDatabase opens a database connection. -func NewSyncServerDatasource(dataSourceName string) (Database, error) { +func NewSyncServerDatasource( + dataSourceName string, + dbProperties common.DbProperties, // nolint:unparam +) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return nil, fmt.Errorf("Cannot use postgres implementation") diff --git a/syncapi/storage/tables/backward_extremities.go b/syncapi/storage/tables/backward_extremities.go new file mode 100644 index 000000000..babd9aaa1 --- /dev/null +++ b/syncapi/storage/tables/backward_extremities.go @@ -0,0 +1,175 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tables + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/common" +) + +// BackwardsExtremitiesStatements contains the SQL statements to implement. +// See BackwardsExtremities to see the parameter and response types. +type BackwardsExtremitiesStatements interface { + Schema() string + InsertBackwardExtremity() string + SelectBackwardExtremitiesForRoom() string + DeleteBackwardExtremity() string +} + +type PostgresBackwardsExtremitiesStatements struct{} + +func (s *PostgresBackwardsExtremitiesStatements) Schema() string { + return `-- Stores output room events received from the roomserver. + CREATE TABLE IF NOT EXISTS syncapi_backward_extremities ( + -- The 'room_id' key for the event. + room_id TEXT NOT NULL, + -- The event ID for the last known event. This is the backwards extremity. + event_id TEXT NOT NULL, + -- The prev_events for the last known event. This is used to update extremities. + prev_event_id TEXT NOT NULL, + + PRIMARY KEY(room_id, event_id, prev_event_id) + ); + ` +} +func (s *PostgresBackwardsExtremitiesStatements) InsertBackwardExtremity() string { + return "" + + "INSERT INTO syncapi_backward_extremities (room_id, event_id, prev_event_id)" + + " VALUES ($1, $2, $3)" + + " ON CONFLICT DO NOTHING" +} +func (s *PostgresBackwardsExtremitiesStatements) SelectBackwardExtremitiesForRoom() string { + return "SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1" +} +func (s *PostgresBackwardsExtremitiesStatements) DeleteBackwardExtremity() string { + return "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" +} + +type SqliteBackwardsExtremitiesStatements struct{} + +func (s *SqliteBackwardsExtremitiesStatements) Schema() string { + return `-- Stores output room events received from the roomserver. +CREATE TABLE IF NOT EXISTS syncapi_backward_extremities ( + -- The 'room_id' key for the event. + room_id TEXT NOT NULL, + -- The event ID for the last known event. This is the backwards extremity. + event_id TEXT NOT NULL, + -- The prev_events for the last known event. This is used to update extremities. + prev_event_id TEXT NOT NULL, + + PRIMARY KEY(room_id, event_id, prev_event_id) +); +` +} + +func (s *SqliteBackwardsExtremitiesStatements) InsertBackwardExtremity() string { + return "" + + "INSERT INTO syncapi_backward_extremities (room_id, event_id, prev_event_id)" + + " VALUES ($1, $2, $3)" + + " ON CONFLICT (room_id, event_id, prev_event_id) DO NOTHING" +} + +func (s *SqliteBackwardsExtremitiesStatements) SelectBackwardExtremitiesForRoom() string { + return "" + + "SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1" +} + +func (s *SqliteBackwardsExtremitiesStatements) DeleteBackwardExtremity() string { + return "" + + "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" +} + +// BackwardsExtremities keeps track of backwards extremities for a room. +// Backwards extremities are the earliest (DAG-wise) known events which we have +// the entire event JSON. These event IDs are used in federation requests to fetch +// even earlier events. +// +// We persist the previous event IDs as well, one per row, so when we do fetch even +// earlier events we can simply delete rows which referenced it. Consider the graph: +// A +// | Event C has 1 prev_event ID: A. +// B C +// |___| Event D has 2 prev_event IDs: B and C. +// | +// D +// The earliest known event we have is D, so this table has 2 rows. +// A backfill request gives us C but not B. We delete rows where prev_event=C. This +// still means that D is a backwards extremity as we do not have event B. However, event +// C is *also* a backwards extremity at this point as we do not have event A. Later, +// when we fetch event B, we delete rows where prev_event=B. This then removes D as +// a backwards extremity because there are no more rows with event_id=B. +type BackwardsExtremities struct { + insertBackwardExtremityStmt *sql.Stmt + selectBackwardExtremitiesForRoomStmt *sql.Stmt + deleteBackwardExtremityStmt *sql.Stmt +} + +// NewBackwardsExtremities prepares the table +func NewBackwardsExtremities(db *sql.DB, stmts BackwardsExtremitiesStatements) (table BackwardsExtremities, err error) { + _, err = db.Exec(stmts.Schema()) + if err != nil { + return + } + if table.insertBackwardExtremityStmt, err = db.Prepare(stmts.InsertBackwardExtremity()); err != nil { + return + } + if table.selectBackwardExtremitiesForRoomStmt, err = db.Prepare(stmts.SelectBackwardExtremitiesForRoom()); err != nil { + return + } + if table.deleteBackwardExtremityStmt, err = db.Prepare(stmts.DeleteBackwardExtremity()); err != nil { + return + } + return +} + +// InsertsBackwardExtremity inserts a new backwards extremity. +func (s *BackwardsExtremities) InsertsBackwardExtremity( + ctx context.Context, txn *sql.Tx, roomID, eventID string, prevEventID string, +) (err error) { + _, err = txn.Stmt(s.insertBackwardExtremityStmt).ExecContext(ctx, roomID, eventID, prevEventID) + return +} + +// SelectBackwardExtremitiesForRoom retrieves all backwards extremities for the room. +func (s *BackwardsExtremities) SelectBackwardExtremitiesForRoom( + ctx context.Context, roomID string, +) (eventIDs []string, err error) { + rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID) + if err != nil { + return + } + defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") + + for rows.Next() { + var eID string + if err = rows.Scan(&eID); err != nil { + return + } + + eventIDs = append(eventIDs, eID) + } + + return eventIDs, rows.Err() +} + +// DeleteBackwardExtremity removes a backwards extremity for a room, if one existed. +func (s *BackwardsExtremities) DeleteBackwardExtremity( + ctx context.Context, txn *sql.Tx, roomID, knownEventID string, +) (err error) { + _, err = txn.Stmt(s.deleteBackwardExtremityStmt).ExecContext(ctx, roomID, knownEventID) + return +} diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index 1535d2b13..4219d5603 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -38,11 +38,11 @@ func SetupSyncAPIComponent( base *basecomponent.BaseDendrite, deviceDB devices.Database, accountsDB accounts.Database, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, federation *gomatrixserverlib.FederationClient, cfg *config.Dendrite, ) { - syncDB, err := storage.NewSyncServerDatasource(string(base.Cfg.Database.SyncAPI)) + syncDB, err := storage.NewSyncServerDatasource(string(base.Cfg.Database.SyncAPI), base.Cfg.DbProperties()) if err != nil { logrus.WithError(err).Panicf("failed to connect to sync db") } @@ -61,7 +61,7 @@ func SetupSyncAPIComponent( requestPool := sync.NewRequestPool(syncDB, notifier, accountsDB) roomConsumer := consumers.NewOutputRoomEventConsumer( - base.Cfg, base.KafkaConsumer, notifier, syncDB, queryAPI, + base.Cfg, base.KafkaConsumer, notifier, syncDB, rsAPI, ) if err = roomConsumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start room server consumer") @@ -81,5 +81,5 @@ func SetupSyncAPIComponent( logrus.WithError(err).Panicf("failed to start typing server consumer") } - routing.Setup(base.APIMux, requestPool, syncDB, deviceDB, federation, queryAPI, cfg) + routing.Setup(base.APIMux, requestPool, syncDB, deviceDB, federation, rsAPI, cfg) } diff --git a/syncapi/types/types.go b/syncapi/types/types.go index 718906ecd..c04fe5219 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/tidwall/gjson" ) var ( @@ -63,8 +64,14 @@ const ( // /sync or /messages, for example. type PaginationToken struct { //Position StreamPosition - Type PaginationTokenType - PDUPosition StreamPosition + Type PaginationTokenType + // For /sync, this is the PDU position. For /messages, this is the topological position (depth). + // TODO: Given how different the positions are depending on the token type, they should probably be renamed + // or use different structs altogether. + PDUPosition StreamPosition + // For /sync, this is the EDU position. For /messages, this is the stream (PDU) position. + // TODO: Given how different the positions are depending on the token type, they should probably be renamed + // or use different structs altogether. EDUTypingPosition StreamPosition } @@ -247,14 +254,17 @@ func NewJoinResponse() *JoinResponse { // InviteResponse represents a /sync response for a room which is under the 'invite' key. type InviteResponse struct { InviteState struct { - Events []gomatrixserverlib.ClientEvent `json:"events"` + Events json.RawMessage `json:"events"` } `json:"invite_state"` } // NewInviteResponse creates an empty response with initialised arrays. -func NewInviteResponse() *InviteResponse { +func NewInviteResponse(event gomatrixserverlib.HeaderedEvent) *InviteResponse { res := InviteResponse{} - res.InviteState.Events = make([]gomatrixserverlib.ClientEvent, 0) + res.InviteState.Events = json.RawMessage{'[', ']'} + if inviteRoomState := gjson.GetBytes(event.Unsigned(), "invite_room_state"); inviteRoomState.Exists() { + res.InviteState.Events = json.RawMessage(inviteRoomState.Raw) + } return &res } diff --git a/sytest-whitelist b/sytest-whitelist index 7bd2a63c4..c957021cb 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -253,3 +253,14 @@ User can invite local user to room with version 3 User can invite local user to room with version 4 A pair of servers can establish a join in a v2 room Can logout all devices +State from remote users is included in the timeline in an incremental sync +User can invite remote user to room with version 1 +User can invite remote user to room with version 2 +User can invite remote user to room with version 3 +User can invite remote user to room with version 4 +User can create and send/receive messages in a room with version 5 +local user can join room with version 5 +User can invite local user to room with version 5 +remote user can join room with version 5 +User can invite remote user to room with version 5 +Remote user can backfill in a room with version 5