diff --git a/appservice/api/query.go b/appservice/api/query.go index 6fcb20890..0a5cc9f1d 100644 --- a/appservice/api/query.go +++ b/appservice/api/query.go @@ -23,9 +23,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/gomatrixserverlib" - - "github.com/matrix-org/dendrite/internal" ) // RoomAliasExistsRequest is a request to an application service @@ -110,7 +109,7 @@ func RetrieveUserProfile( // If no user exists, return if !userResp.UserIDExists { - return nil, internal.ErrProfileNoExists + return nil, eventutil.ErrProfileNoExists } // Try to query the user from the local database again diff --git a/appservice/appservice.go b/appservice/appservice.go index b7adf755b..bd261ff9b 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -31,9 +31,9 @@ import ( "github.com/matrix-org/dendrite/appservice/workers" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/setup" + "github.com/matrix-org/dendrite/internal/sqlutil" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/sirupsen/logrus" ) @@ -46,7 +46,7 @@ func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceQuer // NewInternalAPI returns a concerete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( - base *basecomponent.BaseDendrite, + base *setup.BaseDendrite, accountsDB accounts.Database, deviceDB devices.Database, rsAPI roomserverAPI.RoomserverInternalAPI, @@ -86,12 +86,16 @@ func NewInternalAPI( Cfg: base.Cfg, } - consumer := consumers.NewOutputRoomEventConsumer( - base.Cfg, base.KafkaConsumer, accountsDB, appserviceDB, - rsAPI, workerStates, - ) - if err := consumer.Start(); err != nil { - logrus.WithError(err).Panicf("failed to start appservice roomserver consumer") + // Only consume if we actually have ASes to track, else we'll just chew cycles needlessly. + // We can't add ASes at runtime so this is safe to do. + if len(workerStates) > 0 { + consumer := consumers.NewOutputRoomEventConsumer( + base.Cfg, base.KafkaConsumer, accountsDB, appserviceDB, + rsAPI, workerStates, + ) + if err := consumer.Start(); err != nil { + logrus.WithError(err).Panicf("failed to start appservice roomserver consumer") + } } // Create application service transaction workers @@ -114,7 +118,7 @@ func generateAppServiceAccount( // Create an account for the application service _, err := accountsDB.CreateAccount(ctx, as.SenderLocalpart, "", as.ID) if err != nil { - if errors.Is(err, internal.ErrUserExists) { // This account already exists + if errors.Is(err, sqlutil.ErrUserExists) { // This account already exists return nil } return err diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index bb4df7906..1657fe542 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -91,60 +91,13 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { return nil } - ev := output.NewRoomEvent.Event - log.WithFields(log.Fields{ - "event_id": ev.EventID(), - "room_id": ev.RoomID(), - "type": ev.Type(), - }).Info("appservice received an event from roomserver") - - missingEvents, err := s.lookupMissingStateEvents(output.NewRoomEvent.AddsStateEventIDs, ev) - if err != nil { - return err - } - events := append(missingEvents, ev) + events := []gomatrixserverlib.HeaderedEvent{output.NewRoomEvent.Event} + events = append(events, output.NewRoomEvent.AddStateEvents...) // Send event to any relevant application services return s.filterRoomserverEvents(context.TODO(), events) } -// lookupMissingStateEvents looks up the state events that are added by a new event, -// and returns any not already present. -func (s *OutputRoomEventConsumer) lookupMissingStateEvents( - addsStateEventIDs []string, event gomatrixserverlib.HeaderedEvent, -) ([]gomatrixserverlib.HeaderedEvent, error) { - // Fast path if there aren't any new state events. - if len(addsStateEventIDs) == 0 { - return []gomatrixserverlib.HeaderedEvent{}, nil - } - - // Fast path if the only state event added is the event itself. - if len(addsStateEventIDs) == 1 && addsStateEventIDs[0] == event.EventID() { - return []gomatrixserverlib.HeaderedEvent{}, nil - } - - result := []gomatrixserverlib.HeaderedEvent{} - missing := []string{} - for _, id := range addsStateEventIDs { - if id != event.EventID() { - // If the event isn't the current one, add it to the list of events - // to retrieve from the roomserver - missing = append(missing, id) - } - } - - // Request the missing events from the roomserver - eventReq := api.QueryEventsByIDRequest{EventIDs: missing} - var eventResp api.QueryEventsByIDResponse - if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { - return nil, err - } - - result = append(result, eventResp.Events...) - - return result, nil -} - // filterRoomserverEvents takes in events and decides whether any of them need // to be passed on to an external application service. It does this by checking // each namespace of each registered application service, and if there is a diff --git a/appservice/inthttp/client.go b/appservice/inthttp/client.go index acbd1211b..7e3cb208f 100644 --- a/appservice/inthttp/client.go +++ b/appservice/inthttp/client.go @@ -6,7 +6,7 @@ import ( "net/http" "github.com/matrix-org/dendrite/appservice/api" - internalHTTP "github.com/matrix-org/dendrite/internal/http" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/opentracing/opentracing-go" ) @@ -46,7 +46,7 @@ func (h *httpAppServiceQueryAPI) RoomAliasExists( defer span.Finish() apiURL := h.appserviceURL + AppServiceRoomAliasExistsPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // UserIDExists implements AppServiceQueryAPI @@ -59,5 +59,5 @@ func (h *httpAppServiceQueryAPI) UserIDExists( defer span.Finish() apiURL := h.appserviceURL + AppServiceUserIDExistsPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } diff --git a/appservice/inthttp/server.go b/appservice/inthttp/server.go index 1900635a8..009b7b5db 100644 --- a/appservice/inthttp/server.go +++ b/appservice/inthttp/server.go @@ -6,7 +6,7 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/appservice/api" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/util" ) @@ -14,7 +14,7 @@ import ( func AddRoutes(a api.AppServiceQueryAPI, internalAPIMux *mux.Router) { internalAPIMux.Handle( AppServiceRoomAliasExistsPath, - internal.MakeInternalAPI("appserviceRoomAliasExists", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("appserviceRoomAliasExists", func(req *http.Request) util.JSONResponse { var request api.RoomAliasExistsRequest var response api.RoomAliasExistsResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -28,7 +28,7 @@ func AddRoutes(a api.AppServiceQueryAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( AppServiceUserIDExistsPath, - internal.MakeInternalAPI("appserviceUserIDExists", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("appserviceUserIDExists", func(req *http.Request) util.JSONResponse { var request api.UserIDExistsRequest var response api.UserIDExistsResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { diff --git a/appservice/storage/postgres/storage.go b/appservice/storage/postgres/storage.go index aeead6ed3..3e12f3a0d 100644 --- a/appservice/storage/postgres/storage.go +++ b/appservice/storage/postgres/storage.go @@ -21,7 +21,6 @@ import ( // Import postgres database driver _ "github.com/lib/pq" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -34,7 +33,7 @@ type Database struct { } // NewDatabase opens a new database -func NewDatabase(dataSourceName string, dbProperties internal.DbProperties) (*Database, error) { +func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database, error) { var result Database var err error if result.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { diff --git a/appservice/storage/sqlite3/storage.go b/appservice/storage/sqlite3/storage.go index 2238b3ff9..44dcba4ed 100644 --- a/appservice/storage/sqlite3/storage.go +++ b/appservice/storage/sqlite3/storage.go @@ -20,7 +20,6 @@ import ( "database/sql" // Import SQLite database driver - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" _ "github.com/mattn/go-sqlite3" @@ -41,7 +40,7 @@ func NewDatabase(dataSourceName string) (*Database, error) { if err != nil { return nil, err } - if result.db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil { + if result.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/appservice/storage/storage.go b/appservice/storage/storage.go index 51ba1de53..c848d15d7 100644 --- a/appservice/storage/storage.go +++ b/appservice/storage/storage.go @@ -21,12 +21,12 @@ import ( "github.com/matrix-org/dendrite/appservice/storage/postgres" "github.com/matrix-org/dendrite/appservice/storage/sqlite3" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" ) // NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) // and sets DB connection parameters -func NewDatabase(dataSourceName string, dbProperties internal.DbProperties) (Database, error) { +func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return postgres.NewDatabase(dataSourceName, dbProperties) diff --git a/appservice/storage/storage_wasm.go b/appservice/storage/storage_wasm.go index a6144b435..1d6c4b4a9 100644 --- a/appservice/storage/storage_wasm.go +++ b/appservice/storage/storage_wasm.go @@ -19,12 +19,12 @@ import ( "net/url" "github.com/matrix-org/dendrite/appservice/storage/sqlite3" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" ) func NewDatabase( dataSourceName string, - dbProperties internal.DbProperties, // nolint:unparam + dbProperties sqlutil.DbProperties, // nolint:unparam ) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { diff --git a/clientapi/auth/storage/accounts/postgres/account_data_table.go b/clientapi/auth/storage/accounts/postgres/account_data_table.go index 2e044b367..2f16c5c02 100644 --- a/clientapi/auth/storage/accounts/postgres/account_data_table.go +++ b/clientapi/auth/storage/accounts/postgres/account_data_table.go @@ -19,7 +19,6 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/gomatrixserverlib" ) diff --git a/clientapi/auth/storage/accounts/postgres/membership_table.go b/clientapi/auth/storage/accounts/postgres/membership_table.go index d006e916a..623530acc 100644 --- a/clientapi/auth/storage/accounts/postgres/membership_table.go +++ b/clientapi/auth/storage/accounts/postgres/membership_table.go @@ -18,10 +18,9 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/internal" - "github.com/lib/pq" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/internal" ) const membershipSchema = ` diff --git a/clientapi/auth/storage/accounts/postgres/storage.go b/clientapi/auth/storage/accounts/postgres/storage.go index 4be5dca94..fcb592aef 100644 --- a/clientapi/auth/storage/accounts/postgres/storage.go +++ b/clientapi/auth/storage/accounts/postgres/storage.go @@ -21,7 +21,6 @@ import ( "strconv" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" "golang.org/x/crypto/bcrypt" @@ -33,7 +32,7 @@ import ( // Database represents an account database type Database struct { db *sql.DB - internal.PartitionOffsetStatements + sqlutil.PartitionOffsetStatements accounts accountsStatements profiles profilesStatements memberships membershipStatements @@ -44,13 +43,13 @@ type Database struct { } // NewDatabase creates a new accounts and profiles database -func NewDatabase(dataSourceName string, dbProperties internal.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) { +func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } - partitions := internal.PartitionOffsetStatements{} + partitions := sqlutil.PartitionOffsetStatements{} if err = partitions.Prepare(db, "account"); err != nil { return nil, err } @@ -123,7 +122,7 @@ func (d *Database) SetDisplayName( // CreateGuestAccount makes a new guest account and creates an empty profile // for this account. func (d *Database) CreateGuestAccount(ctx context.Context) (acc *authtypes.Account, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { var numLocalpart int64 numLocalpart, err = d.accounts.selectNewNumericLocalpart(ctx, txn) if err != nil { @@ -138,11 +137,11 @@ func (d *Database) CreateGuestAccount(ctx context.Context) (acc *authtypes.Accou // CreateAccount makes a new account with the given login name and password, and creates an empty profile // for this account. If no password is supplied, the account will be a passwordless account. If the -// account already exists, it will return nil, ErrUserExists. +// account already exists, it will return nil, sqlutil.ErrUserExists. func (d *Database) CreateAccount( ctx context.Context, localpart, plaintextPassword, appserviceID string, ) (acc *authtypes.Account, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { acc, err = d.createAccount(ctx, txn, localpart, plaintextPassword, appserviceID) return err }) @@ -163,8 +162,8 @@ func (d *Database) createAccount( } } if err := d.profiles.insertProfile(ctx, txn, localpart); err != nil { - if internal.IsUniqueConstraintViolationErr(err) { - return nil, internal.ErrUserExists + if sqlutil.IsUniqueConstraintViolationErr(err) { + return nil, sqlutil.ErrUserExists } return nil, err } @@ -210,7 +209,7 @@ func (d *Database) removeMembershipsByEventIDs( func (d *Database) UpdateMemberships( ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string, ) error { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { if err := d.removeMembershipsByEventIDs(ctx, txn, idsToRemove); err != nil { return err } @@ -297,7 +296,7 @@ func (d *Database) newMembership( func (d *Database) SaveAccountData( ctx context.Context, localpart, roomID, dataType, content string, ) error { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { return d.accountDatas.insertAccountData(ctx, txn, localpart, roomID, dataType, content) }) } @@ -348,7 +347,7 @@ var Err3PIDInUse = errors.New("This third-party identifier is already in use") func (d *Database) SaveThreePIDAssociation( ctx context.Context, threepid, localpart, medium string, ) (err error) { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { user, err := d.threepids.selectLocalpartForThreePID( ctx, txn, threepid, medium, ) diff --git a/clientapi/auth/storage/accounts/postgres/threepid_table.go b/clientapi/auth/storage/accounts/postgres/threepid_table.go index 0b12b5c5b..7de96350c 100644 --- a/clientapi/auth/storage/accounts/postgres/threepid_table.go +++ b/clientapi/auth/storage/accounts/postgres/threepid_table.go @@ -18,7 +18,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" ) @@ -82,7 +82,7 @@ func (s *threepidStatements) prepare(db *sql.DB) (err error) { func (s *threepidStatements) selectLocalpartForThreePID( ctx context.Context, txn *sql.Tx, threepid string, medium string, ) (localpart string, err error) { - stmt := internal.TxStmt(txn, s.selectLocalpartForThreePIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectLocalpartForThreePIDStmt) err = stmt.QueryRowContext(ctx, threepid, medium).Scan(&localpart) if err == sql.ErrNoRows { return "", nil @@ -117,7 +117,7 @@ func (s *threepidStatements) selectThreePIDsForLocalpart( func (s *threepidStatements) insertThreePID( ctx context.Context, txn *sql.Tx, threepid, medium, localpart string, ) (err error) { - stmt := internal.TxStmt(txn, s.insertThreePIDStmt) + stmt := sqlutil.TxStmt(txn, s.insertThreePIDStmt) _, err = stmt.ExecContext(ctx, threepid, medium, localpart) return } diff --git a/clientapi/auth/storage/accounts/sqlite3/membership_table.go b/clientapi/auth/storage/accounts/sqlite3/membership_table.go index 90e16fb01..67958f27d 100644 --- a/clientapi/auth/storage/accounts/sqlite3/membership_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/membership_table.go @@ -21,6 +21,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" ) const membershipSchema = ` @@ -95,7 +96,7 @@ func (s *membershipStatements) insertMembership( func (s *membershipStatements) deleteMembershipsByEventIDs( ctx context.Context, txn *sql.Tx, eventIDs []string, ) (err error) { - sqlStr := strings.Replace(deleteMembershipsByEventIDsSQL, "($1)", internal.QueryVariadic(len(eventIDs)), 1) + sqlStr := strings.Replace(deleteMembershipsByEventIDsSQL, "($1)", sqlutil.QueryVariadic(len(eventIDs)), 1) iEventIDs := make([]interface{}, len(eventIDs)) for i, e := range eventIDs { iEventIDs[i] = e diff --git a/clientapi/auth/storage/accounts/sqlite3/storage.go b/clientapi/auth/storage/accounts/sqlite3/storage.go index 31426e471..44245a99d 100644 --- a/clientapi/auth/storage/accounts/sqlite3/storage.go +++ b/clientapi/auth/storage/accounts/sqlite3/storage.go @@ -22,7 +22,6 @@ import ( "sync" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" "golang.org/x/crypto/bcrypt" @@ -32,7 +31,7 @@ import ( // Database represents an account database type Database struct { db *sql.DB - internal.PartitionOffsetStatements + sqlutil.PartitionOffsetStatements accounts accountsStatements profiles profilesStatements memberships membershipStatements @@ -52,10 +51,10 @@ func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) if err != nil { return nil, err } - if db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil { + if db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { return nil, err } - partitions := internal.PartitionOffsetStatements{} + partitions := sqlutil.PartitionOffsetStatements{} if err = partitions.Prepare(db, "account"); err != nil { return nil, err } @@ -128,7 +127,7 @@ func (d *Database) SetDisplayName( // CreateGuestAccount makes a new guest account and creates an empty profile // for this account. func (d *Database) CreateGuestAccount(ctx context.Context) (acc *authtypes.Account, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { // We need to lock so we sequentially create numeric localparts. If we don't, two calls to // this function will cause the same number to be selected and one will fail with 'database is locked' // when the first txn upgrades to a write txn. @@ -154,7 +153,7 @@ func (d *Database) CreateGuestAccount(ctx context.Context) (acc *authtypes.Accou func (d *Database) CreateAccount( ctx context.Context, localpart, plaintextPassword, appserviceID string, ) (acc *authtypes.Account, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { acc, err = d.createAccount(ctx, txn, localpart, plaintextPassword, appserviceID) return err }) @@ -175,7 +174,7 @@ func (d *Database) createAccount( } if err := d.profiles.insertProfile(ctx, txn, localpart); err != nil { if isConstraintError(err) { - return nil, internal.ErrUserExists + return nil, sqlutil.ErrUserExists } return nil, err } @@ -221,7 +220,7 @@ func (d *Database) removeMembershipsByEventIDs( func (d *Database) UpdateMemberships( ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string, ) error { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { if err := d.removeMembershipsByEventIDs(ctx, txn, idsToRemove); err != nil { return err } @@ -308,7 +307,7 @@ func (d *Database) newMembership( func (d *Database) SaveAccountData( ctx context.Context, localpart, roomID, dataType, content string, ) error { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { return d.accountDatas.insertAccountData(ctx, txn, localpart, roomID, dataType, content) }) } @@ -359,7 +358,7 @@ var Err3PIDInUse = errors.New("This third-party identifier is already in use") func (d *Database) SaveThreePIDAssociation( ctx context.Context, threepid, localpart, medium string, ) (err error) { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { user, err := d.threepids.selectLocalpartForThreePID( ctx, txn, threepid, medium, ) diff --git a/clientapi/auth/storage/accounts/sqlite3/threepid_table.go b/clientapi/auth/storage/accounts/sqlite3/threepid_table.go index f78a5ffcd..0200dee7f 100644 --- a/clientapi/auth/storage/accounts/sqlite3/threepid_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/threepid_table.go @@ -19,6 +19,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" ) @@ -82,7 +83,7 @@ func (s *threepidStatements) prepare(db *sql.DB) (err error) { func (s *threepidStatements) selectLocalpartForThreePID( ctx context.Context, txn *sql.Tx, threepid string, medium string, ) (localpart string, err error) { - stmt := internal.TxStmt(txn, s.selectLocalpartForThreePIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectLocalpartForThreePIDStmt) err = stmt.QueryRowContext(ctx, threepid, medium).Scan(&localpart) if err == sql.ErrNoRows { return "", nil @@ -117,7 +118,7 @@ func (s *threepidStatements) selectThreePIDsForLocalpart( func (s *threepidStatements) insertThreePID( ctx context.Context, txn *sql.Tx, threepid, medium, localpart string, ) (err error) { - stmt := internal.TxStmt(txn, s.insertThreePIDStmt) + stmt := sqlutil.TxStmt(txn, s.insertThreePIDStmt) _, err = stmt.ExecContext(ctx, threepid, medium, localpart) return } diff --git a/clientapi/auth/storage/accounts/storage.go b/clientapi/auth/storage/accounts/storage.go index 37126b30b..42ec14fc4 100644 --- a/clientapi/auth/storage/accounts/storage.go +++ b/clientapi/auth/storage/accounts/storage.go @@ -21,13 +21,13 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/postgres" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) // NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) // and sets postgres connection parameters -func NewDatabase(dataSourceName string, dbProperties internal.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) { +func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return postgres.NewDatabase(dataSourceName, dbProperties, serverName) diff --git a/clientapi/auth/storage/accounts/storage_wasm.go b/clientapi/auth/storage/accounts/storage_wasm.go index 81e77cf79..6c221ccf5 100644 --- a/clientapi/auth/storage/accounts/storage_wasm.go +++ b/clientapi/auth/storage/accounts/storage_wasm.go @@ -19,13 +19,13 @@ import ( "net/url" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) func NewDatabase( dataSourceName string, - dbProperties internal.DbProperties, // nolint:unparam + dbProperties sqlutil.DbProperties, // nolint:unparam serverName gomatrixserverlib.ServerName, ) (Database, error) { uri, err := url.Parse(dataSourceName) diff --git a/clientapi/auth/storage/devices/postgres/devices_table.go b/clientapi/auth/storage/devices/postgres/devices_table.go index c84e83d3d..149ca659f 100644 --- a/clientapi/auth/storage/devices/postgres/devices_table.go +++ b/clientapi/auth/storage/devices/postgres/devices_table.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -137,7 +138,7 @@ func (s *devicesStatements) insertDevice( ) (*authtypes.Device, error) { createdTimeMS := time.Now().UnixNano() / 1000000 var sessionID int64 - stmt := internal.TxStmt(txn, s.insertDeviceStmt) + stmt := sqlutil.TxStmt(txn, s.insertDeviceStmt) if err := stmt.QueryRowContext(ctx, id, localpart, accessToken, createdTimeMS, displayName).Scan(&sessionID); err != nil { return nil, err } @@ -153,7 +154,7 @@ func (s *devicesStatements) insertDevice( func (s *devicesStatements) deleteDevice( ctx context.Context, txn *sql.Tx, id, localpart string, ) error { - stmt := internal.TxStmt(txn, s.deleteDeviceStmt) + stmt := sqlutil.TxStmt(txn, s.deleteDeviceStmt) _, err := stmt.ExecContext(ctx, id, localpart) return err } @@ -163,7 +164,7 @@ func (s *devicesStatements) deleteDevice( func (s *devicesStatements) deleteDevices( ctx context.Context, txn *sql.Tx, localpart string, devices []string, ) error { - stmt := internal.TxStmt(txn, s.deleteDevicesStmt) + stmt := sqlutil.TxStmt(txn, s.deleteDevicesStmt) _, err := stmt.ExecContext(ctx, localpart, pq.Array(devices)) return err } @@ -173,7 +174,7 @@ func (s *devicesStatements) deleteDevices( func (s *devicesStatements) deleteDevicesByLocalpart( ctx context.Context, txn *sql.Tx, localpart string, ) error { - stmt := internal.TxStmt(txn, s.deleteDevicesByLocalpartStmt) + stmt := sqlutil.TxStmt(txn, s.deleteDevicesByLocalpartStmt) _, err := stmt.ExecContext(ctx, localpart) return err } @@ -181,7 +182,7 @@ func (s *devicesStatements) deleteDevicesByLocalpart( func (s *devicesStatements) updateDeviceName( ctx context.Context, txn *sql.Tx, localpart, deviceID string, displayName *string, ) error { - stmt := internal.TxStmt(txn, s.updateDeviceNameStmt) + stmt := sqlutil.TxStmt(txn, s.updateDeviceNameStmt) _, err := stmt.ExecContext(ctx, displayName, localpart, deviceID) return err } diff --git a/clientapi/auth/storage/devices/postgres/storage.go b/clientapi/auth/storage/devices/postgres/storage.go index e54e7c5d7..2b9aede2f 100644 --- a/clientapi/auth/storage/devices/postgres/storage.go +++ b/clientapi/auth/storage/devices/postgres/storage.go @@ -21,7 +21,6 @@ import ( "encoding/base64" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -36,7 +35,7 @@ type Database struct { } // NewDatabase creates a new device database -func NewDatabase(dataSourceName string, dbProperties internal.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) { +func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { @@ -83,7 +82,7 @@ func (d *Database) CreateDevice( displayName *string, ) (dev *authtypes.Device, returnErr error) { if deviceID != nil { - returnErr = internal.WithTransaction(d.db, func(txn *sql.Tx) error { + returnErr = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { var err error // Revoke existing tokens for this device if err = d.devices.deleteDevice(ctx, txn, *deviceID, localpart); err != nil { @@ -103,7 +102,7 @@ func (d *Database) CreateDevice( return } - returnErr = internal.WithTransaction(d.db, func(txn *sql.Tx) error { + returnErr = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { var err error dev, err = d.devices.insertDevice(ctx, txn, newDeviceID, localpart, accessToken, displayName) return err @@ -133,7 +132,7 @@ func generateDeviceID() (string, error) { func (d *Database) UpdateDevice( ctx context.Context, localpart, deviceID string, displayName *string, ) error { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { return d.devices.updateDeviceName(ctx, txn, localpart, deviceID, displayName) }) } @@ -145,7 +144,7 @@ func (d *Database) UpdateDevice( func (d *Database) RemoveDevice( ctx context.Context, deviceID, localpart string, ) error { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { if err := d.devices.deleteDevice(ctx, txn, deviceID, localpart); err != sql.ErrNoRows { return err } @@ -160,7 +159,7 @@ func (d *Database) RemoveDevice( func (d *Database) RemoveDevices( ctx context.Context, localpart string, devices []string, ) error { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { if err := d.devices.deleteDevices(ctx, txn, localpart, devices); err != sql.ErrNoRows { return err } @@ -174,7 +173,7 @@ func (d *Database) RemoveDevices( func (d *Database) RemoveAllDevices( ctx context.Context, localpart string, ) error { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { if err := d.devices.deleteDevicesByLocalpart(ctx, txn, localpart); err != sql.ErrNoRows { return err } diff --git a/clientapi/auth/storage/devices/sqlite3/devices_table.go b/clientapi/auth/storage/devices/sqlite3/devices_table.go index 49f3eaed7..4656b0041 100644 --- a/clientapi/auth/storage/devices/sqlite3/devices_table.go +++ b/clientapi/auth/storage/devices/sqlite3/devices_table.go @@ -20,7 +20,7 @@ import ( "strings" "time" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/userutil" @@ -128,8 +128,8 @@ func (s *devicesStatements) insertDevice( ) (*authtypes.Device, error) { createdTimeMS := time.Now().UnixNano() / 1000000 var sessionID int64 - countStmt := internal.TxStmt(txn, s.selectDevicesCountStmt) - insertStmt := internal.TxStmt(txn, s.insertDeviceStmt) + countStmt := sqlutil.TxStmt(txn, s.selectDevicesCountStmt) + insertStmt := sqlutil.TxStmt(txn, s.insertDeviceStmt) if err := countStmt.QueryRowContext(ctx).Scan(&sessionID); err != nil { return nil, err } @@ -148,7 +148,7 @@ func (s *devicesStatements) insertDevice( func (s *devicesStatements) deleteDevice( ctx context.Context, txn *sql.Tx, id, localpart string, ) error { - stmt := internal.TxStmt(txn, s.deleteDeviceStmt) + stmt := sqlutil.TxStmt(txn, s.deleteDeviceStmt) _, err := stmt.ExecContext(ctx, id, localpart) return err } @@ -156,12 +156,12 @@ func (s *devicesStatements) deleteDevice( func (s *devicesStatements) deleteDevices( ctx context.Context, txn *sql.Tx, localpart string, devices []string, ) error { - orig := strings.Replace(deleteDevicesSQL, "($1)", internal.QueryVariadic(len(devices)), 1) + orig := strings.Replace(deleteDevicesSQL, "($1)", sqlutil.QueryVariadic(len(devices)), 1) prep, err := s.db.Prepare(orig) if err != nil { return err } - stmt := internal.TxStmt(txn, prep) + stmt := sqlutil.TxStmt(txn, prep) params := make([]interface{}, len(devices)+1) params[0] = localpart for i, v := range devices { @@ -175,7 +175,7 @@ func (s *devicesStatements) deleteDevices( func (s *devicesStatements) deleteDevicesByLocalpart( ctx context.Context, txn *sql.Tx, localpart string, ) error { - stmt := internal.TxStmt(txn, s.deleteDevicesByLocalpartStmt) + stmt := sqlutil.TxStmt(txn, s.deleteDevicesByLocalpartStmt) _, err := stmt.ExecContext(ctx, localpart) return err } @@ -183,7 +183,7 @@ func (s *devicesStatements) deleteDevicesByLocalpart( func (s *devicesStatements) updateDeviceName( ctx context.Context, txn *sql.Tx, localpart, deviceID string, displayName *string, ) error { - stmt := internal.TxStmt(txn, s.updateDeviceNameStmt) + stmt := sqlutil.TxStmt(txn, s.updateDeviceNameStmt) _, err := stmt.ExecContext(ctx, displayName, localpart, deviceID) return err } diff --git a/clientapi/auth/storage/devices/sqlite3/storage.go b/clientapi/auth/storage/devices/sqlite3/storage.go index e05a53b46..09e0bc81e 100644 --- a/clientapi/auth/storage/devices/sqlite3/storage.go +++ b/clientapi/auth/storage/devices/sqlite3/storage.go @@ -21,7 +21,6 @@ import ( "encoding/base64" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" @@ -45,7 +44,7 @@ func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) if err != nil { return nil, err } - if db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil { + if db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { return nil, err } d := devicesStatements{} @@ -89,7 +88,7 @@ func (d *Database) CreateDevice( displayName *string, ) (dev *authtypes.Device, returnErr error) { if deviceID != nil { - returnErr = internal.WithTransaction(d.db, func(txn *sql.Tx) error { + returnErr = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { var err error // Revoke existing tokens for this device if err = d.devices.deleteDevice(ctx, txn, *deviceID, localpart); err != nil { @@ -109,7 +108,7 @@ func (d *Database) CreateDevice( return } - returnErr = internal.WithTransaction(d.db, func(txn *sql.Tx) error { + returnErr = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { var err error dev, err = d.devices.insertDevice(ctx, txn, newDeviceID, localpart, accessToken, displayName) return err @@ -139,7 +138,7 @@ func generateDeviceID() (string, error) { func (d *Database) UpdateDevice( ctx context.Context, localpart, deviceID string, displayName *string, ) error { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { return d.devices.updateDeviceName(ctx, txn, localpart, deviceID, displayName) }) } @@ -151,7 +150,7 @@ func (d *Database) UpdateDevice( func (d *Database) RemoveDevice( ctx context.Context, deviceID, localpart string, ) error { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { if err := d.devices.deleteDevice(ctx, txn, deviceID, localpart); err != sql.ErrNoRows { return err } @@ -166,7 +165,7 @@ func (d *Database) RemoveDevice( func (d *Database) RemoveDevices( ctx context.Context, localpart string, devices []string, ) error { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { if err := d.devices.deleteDevices(ctx, txn, localpart, devices); err != sql.ErrNoRows { return err } @@ -180,7 +179,7 @@ func (d *Database) RemoveDevices( func (d *Database) RemoveAllDevices( ctx context.Context, localpart string, ) error { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { if err := d.devices.deleteDevicesByLocalpart(ctx, txn, localpart); err != sql.ErrNoRows { return err } diff --git a/clientapi/auth/storage/devices/storage.go b/clientapi/auth/storage/devices/storage.go index c132598bd..d0d203427 100644 --- a/clientapi/auth/storage/devices/storage.go +++ b/clientapi/auth/storage/devices/storage.go @@ -21,13 +21,13 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/postgres" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) // NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) // and sets postgres connection parameters -func NewDatabase(dataSourceName string, dbProperties internal.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) { +func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return postgres.NewDatabase(dataSourceName, dbProperties, serverName) diff --git a/clientapi/auth/storage/devices/storage_wasm.go b/clientapi/auth/storage/devices/storage_wasm.go index 14c19e74b..e32471d8c 100644 --- a/clientapi/auth/storage/devices/storage_wasm.go +++ b/clientapi/auth/storage/devices/storage_wasm.go @@ -19,13 +19,13 @@ import ( "net/url" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) func NewDatabase( dataSourceName string, - dbProperties internal.DbProperties, // nolint:unparam + dbProperties sqlutil.DbProperties, // nolint:unparam serverName gomatrixserverlib.ServerName, ) (Database, error) { uri, err := url.Parse(dataSourceName) diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 2780f367c..545b95b0e 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -41,7 +41,6 @@ func AddPublicRoutes( deviceDB devices.Database, accountsDB accounts.Database, federation *gomatrixserverlib.FederationClient, - keyRing *gomatrixserverlib.KeyRing, rsAPI roomserverAPI.RoomserverInternalAPI, eduInputAPI eduServerAPI.EDUServerInputAPI, asAPI appserviceAPI.AppServiceQueryAPI, @@ -62,7 +61,7 @@ func AddPublicRoutes( routing.Setup( router, cfg, eduInputAPI, rsAPI, asAPI, - accountsDB, deviceDB, federation, *keyRing, + accountsDB, deviceDB, federation, syncProducer, transactionsCache, fsAPI, ) } diff --git a/clientapi/consumers/roomserver.go b/clientapi/consumers/roomserver.go index bd8ac1dcd..caa028ba3 100644 --- a/clientapi/consumers/roomserver.go +++ b/clientapi/consumers/roomserver.go @@ -84,63 +84,9 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { return nil } - ev := output.NewRoomEvent.Event - log.WithFields(log.Fields{ - "event_id": ev.EventID(), - "room_id": ev.RoomID(), - "type": ev.Type(), - }).Info("received event from roomserver") - - events, err := s.lookupStateEvents(output.NewRoomEvent.AddsStateEventIDs, ev.Event) - if err != nil { - return err - } - - return s.db.UpdateMemberships(context.TODO(), events, output.NewRoomEvent.RemovesStateEventIDs) -} - -// lookupStateEvents looks up the state events that are added by a new event. -func (s *OutputRoomEventConsumer) lookupStateEvents( - addsStateEventIDs []string, event gomatrixserverlib.Event, -) ([]gomatrixserverlib.Event, error) { - // Fast path if there aren't any new state events. - if len(addsStateEventIDs) == 0 { - // If the event is a membership update (e.g. for a profile update), it won't - // show up in AddsStateEventIDs, so we need to add it manually - if event.Type() == "m.room.member" { - return []gomatrixserverlib.Event{event}, nil - } - return nil, nil - } - - // Fast path if the only state event added is the event itself. - if len(addsStateEventIDs) == 1 && addsStateEventIDs[0] == event.EventID() { - return []gomatrixserverlib.Event{event}, nil - } - - result := []gomatrixserverlib.Event{} - missing := []string{} - for _, id := range addsStateEventIDs { - // Append the current event in the results if its ID is in the events list - if id == event.EventID() { - result = append(result, event) - } else { - // If the event isn't the current one, add it to the list of events - // to retrieve from the roomserver - missing = append(missing, id) - } - } - - // Request the missing events from the roomserver - eventReq := api.QueryEventsByIDRequest{EventIDs: missing} - var eventResp api.QueryEventsByIDResponse - if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { - return nil, err - } - - for _, headeredEvent := range eventResp.Events { - result = append(result, headeredEvent.Event) - } - - return result, nil + return s.db.UpdateMemberships( + context.TODO(), + gomatrixserverlib.UnwrapEventHeaders(output.NewRoomEvent.AddsState()), + output.NewRoomEvent.RemovesStateEventIDs, + ) } diff --git a/clientapi/producers/syncapi.go b/clientapi/producers/syncapi.go index 244a61dc2..6ab8eef28 100644 --- a/clientapi/producers/syncapi.go +++ b/clientapi/producers/syncapi.go @@ -17,9 +17,9 @@ package producers import ( "encoding/json" - "github.com/matrix-org/dendrite/internal" - "github.com/Shopify/sarama" + "github.com/matrix-org/dendrite/internal/eventutil" + log "github.com/sirupsen/logrus" ) // SyncAPIProducer produces events for the sync API server to consume @@ -32,7 +32,7 @@ type SyncAPIProducer struct { func (p *SyncAPIProducer) SendData(userID string, roomID string, dataType string) error { var m sarama.ProducerMessage - data := internal.AccountData{ + data := eventutil.AccountData{ RoomID: roomID, Type: dataType, } @@ -44,6 +44,11 @@ func (p *SyncAPIProducer) SendData(userID string, roomID string, dataType string m.Topic = string(p.Topic) m.Key = sarama.StringEncoder(userID) m.Value = sarama.ByteEncoder(value) + log.WithFields(log.Fields{ + "user_id": userID, + "room_id": roomID, + "data_type": dataType, + }).Infof("Producing to topic '%s'", p.Topic) _, _, err = p.Producer.SendMessage(&m) return err diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 89f49d356..fd91a1060 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -30,8 +30,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/threepid" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" log "github.com/sirupsen/logrus" @@ -98,7 +98,7 @@ func (r createRoomRequest) Validate() *util.JSONResponse { // Validate creation_content fields defined in the spec by marshalling the // creation_content map into bytes and then unmarshalling the bytes into - // internal.CreateContent. + // eventutil.CreateContent. creationContentBytes, err := json.Marshal(r.CreationContent) if err != nil { @@ -279,25 +279,25 @@ func createRoom( eventsToMake := []fledglingEvent{ {"m.room.create", "", r.CreationContent}, {"m.room.member", userID, membershipContent}, - {"m.room.power_levels", "", internal.InitialPowerLevelsContent(userID)}, + {"m.room.power_levels", "", eventutil.InitialPowerLevelsContent(userID)}, {"m.room.join_rules", "", gomatrixserverlib.JoinRuleContent{JoinRule: joinRules}}, - {"m.room.history_visibility", "", internal.HistoryVisibilityContent{HistoryVisibility: historyVisibility}}, + {"m.room.history_visibility", "", eventutil.HistoryVisibilityContent{HistoryVisibility: historyVisibility}}, } if roomAlias != "" { // TODO: bit of a chicken and egg problem here as the alias doesn't exist and cannot until we have made the room. // This means we might fail creating the alias but say the canonical alias is something that doesn't exist. // m.room.aliases is handled when we call roomserver.SetRoomAlias - eventsToMake = append(eventsToMake, fledglingEvent{"m.room.canonical_alias", "", internal.CanonicalAlias{Alias: roomAlias}}) + eventsToMake = append(eventsToMake, fledglingEvent{"m.room.canonical_alias", "", eventutil.CanonicalAlias{Alias: roomAlias}}) } if r.GuestCanJoin { - eventsToMake = append(eventsToMake, fledglingEvent{"m.room.guest_access", "", internal.GuestAccessContent{GuestAccess: "can_join"}}) + eventsToMake = append(eventsToMake, fledglingEvent{"m.room.guest_access", "", eventutil.GuestAccessContent{GuestAccess: "can_join"}}) } eventsToMake = append(eventsToMake, r.InitialState...) if r.Name != "" { - eventsToMake = append(eventsToMake, fledglingEvent{"m.room.name", "", internal.NameContent{Name: r.Name}}) + eventsToMake = append(eventsToMake, fledglingEvent{"m.room.name", "", eventutil.NameContent{Name: r.Name}}) } if r.Topic != "" { - eventsToMake = append(eventsToMake, fledglingEvent{"m.room.topic", "", internal.TopicContent{Topic: r.Topic}}) + eventsToMake = append(eventsToMake, fledglingEvent{"m.room.topic", "", eventutil.TopicContent{Topic: r.Topic}}) } // TODO: invite events // TODO: 3pid invite events diff --git a/clientapi/routing/getevent.go b/clientapi/routing/getevent.go index 3ca9d1b62..16f36d661 100644 --- a/clientapi/routing/getevent.go +++ b/clientapi/routing/getevent.go @@ -32,7 +32,6 @@ type getEventRequest struct { eventID string cfg *config.Dendrite federation *gomatrixserverlib.FederationClient - keyRing gomatrixserverlib.KeyRing requestedEvent gomatrixserverlib.Event } @@ -46,7 +45,6 @@ func GetEvent( cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, federation *gomatrixserverlib.FederationClient, - keyRing gomatrixserverlib.KeyRing, ) util.JSONResponse { eventsReq := api.QueryEventsByIDRequest{ EventIDs: []string{eventID}, @@ -75,7 +73,6 @@ func GetEvent( eventID: eventID, cfg: cfg, federation: federation, - keyRing: keyRing, requestedEvent: requestedEvent, } diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 0b2c2bc5e..53484a1a6 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -26,8 +26,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/threepid" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -95,7 +95,7 @@ func SendMembership( Code: http.StatusBadRequest, JSON: jsonerror.BadJSON(err.Error()), } - } else if err == internal.ErrRoomNoExists { + } else if err == eventutil.ErrRoomNoExists { return util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound(err.Error()), @@ -188,7 +188,7 @@ func buildMembershipEvent( return nil, err } - return internal.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil) + return eventutil.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil) } // loadProfile lookups the profile of a given user from the database and returns @@ -268,7 +268,7 @@ func checkAndProcessThreepid( Code: http.StatusBadRequest, JSON: jsonerror.NotTrusted(body.IDServer), } - } else if err == internal.ErrRoomNoExists { + } else if err == eventutil.ErrRoomNoExists { return inviteStored, &util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound(err.Error()), diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index 642a72887..a7a82ed58 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -24,8 +24,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -42,7 +42,7 @@ func GetProfile( ) util.JSONResponse { profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation) if err != nil { - if err == internal.ErrProfileNoExists { + if err == eventutil.ErrProfileNoExists { return util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound("The user does not exist or does not have a profile"), @@ -55,7 +55,7 @@ func GetProfile( return util.JSONResponse{ Code: http.StatusOK, - JSON: internal.ProfileResponse{ + JSON: eventutil.ProfileResponse{ AvatarURL: profile.AvatarURL, DisplayName: profile.DisplayName, }, @@ -70,7 +70,7 @@ func GetAvatarURL( ) util.JSONResponse { profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation) if err != nil { - if err == internal.ErrProfileNoExists { + if err == eventutil.ErrProfileNoExists { return util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound("The user does not exist or does not have a profile"), @@ -83,7 +83,7 @@ func GetAvatarURL( return util.JSONResponse{ Code: http.StatusOK, - JSON: internal.AvatarURL{ + JSON: eventutil.AvatarURL{ AvatarURL: profile.AvatarURL, }, } @@ -102,7 +102,7 @@ func SetAvatarURL( } } - var r internal.AvatarURL + var r eventutil.AvatarURL if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil { return *resErr } @@ -184,7 +184,7 @@ func GetDisplayName( ) util.JSONResponse { profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation) if err != nil { - if err == internal.ErrProfileNoExists { + if err == eventutil.ErrProfileNoExists { return util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound("The user does not exist or does not have a profile"), @@ -197,7 +197,7 @@ func GetDisplayName( return util.JSONResponse{ Code: http.StatusOK, - JSON: internal.DisplayName{ + JSON: eventutil.DisplayName{ DisplayName: profile.DisplayName, }, } @@ -216,7 +216,7 @@ func SetDisplayName( } } - var r internal.DisplayName + var r eventutil.DisplayName if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil { return *resErr } @@ -293,7 +293,7 @@ func SetDisplayName( // getProfile gets the full profile of a user by querying the database or a // remote homeserver. // Returns an error when something goes wrong or specifically -// internal.ErrProfileNoExists when the profile doesn't exist. +// eventutil.ErrProfileNoExists when the profile doesn't exist. func getProfile( ctx context.Context, accountDB accounts.Database, cfg *config.Dendrite, userID string, @@ -310,7 +310,7 @@ func getProfile( if fedErr != nil { if x, ok := fedErr.(gomatrix.HTTPError); ok { if x.Code == http.StatusNotFound { - return nil, internal.ErrProfileNoExists + return nil, eventutil.ErrProfileNoExists } } @@ -365,7 +365,7 @@ func buildMembershipEvents( return nil, err } - event, err := internal.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil) + event, err := eventutil.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil) if err != nil { return nil, err } diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index d356db2cb..8988dbd05 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -33,6 +33,8 @@ import ( "time" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/eventutil" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/clientapi/auth" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" @@ -41,7 +43,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/userutil" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/tokens" "github.com/matrix-org/util" @@ -136,7 +137,7 @@ type registerRequest struct { DeviceID *string `json:"device_id"` // Prevent this user from logging in - InhibitLogin internal.WeakBoolean `json:"inhibit_login"` + InhibitLogin eventutil.WeakBoolean `json:"inhibit_login"` // Application Services place Type in the root of their registration // request, whereas clients place it in the authDict struct. @@ -811,7 +812,7 @@ func completeRegistration( accountDB accounts.Database, deviceDB devices.Database, username, password, appserviceID string, - inhibitLogin internal.WeakBoolean, + inhibitLogin eventutil.WeakBoolean, displayName, deviceID *string, ) util.JSONResponse { if username == "" { @@ -830,7 +831,7 @@ func completeRegistration( acc, err := accountDB.CreateAccount(ctx, username, password, appserviceID) if err != nil { - if errors.Is(err, internal.ErrUserExists) { // user already exists + if errors.Is(err, sqlutil.ErrUserExists) { // user already exists return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UserInUse("Desired user ID is already taken."), diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 024707759..82a80fff9 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -29,8 +29,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/producers" eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/transactions" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -55,14 +55,13 @@ func Setup( accountDB accounts.Database, deviceDB devices.Database, federation *gomatrixserverlib.FederationClient, - keyRing gomatrixserverlib.KeyRing, syncProducer *producers.SyncAPIProducer, transactionsCache *transactions.Cache, federationSender federationSenderAPI.FederationSenderInternalAPI, ) { publicAPIMux.Handle("/client/versions", - internal.MakeExternalAPI("versions", func(req *http.Request) util.JSONResponse { + httputil.MakeExternalAPI("versions", func(req *http.Request) util.JSONResponse { return util.JSONResponse{ Code: http.StatusOK, JSON: struct { @@ -88,13 +87,13 @@ func Setup( } r0mux.Handle("/createRoom", - internal.MakeAuthAPI("createRoom", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("createRoom", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return CreateRoom(req, device, cfg, accountDB, rsAPI, asAPI) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/join/{roomIDOrAlias}", - internal.MakeAuthAPI(gomatrixserverlib.Join, authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI(gomatrixserverlib.Join, authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -104,13 +103,13 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/joined_rooms", - internal.MakeAuthAPI("joined_rooms", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("joined_rooms", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return GetJoinedRooms(req, device, accountDB) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/leave", - internal.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -120,8 +119,8 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/{membership:(?:join|kick|ban|unban|invite)}", - internal.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -129,8 +128,8 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/send/{eventType}", - internal.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -138,8 +137,8 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}", - internal.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -149,33 +148,33 @@ func Setup( }), ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/event/{eventID}", - internal.MakeAuthAPI("rooms_get_event", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("rooms_get_event", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } - return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI, federation, keyRing) + return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI, federation) }), ).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/state", internal.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + r0mux.Handle("/rooms/{roomID}/state", httputil.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } return OnIncomingStateRequest(req.Context(), rsAPI, vars["roomID"]) })).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/state/{type}", internal.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + r0mux.Handle("/rooms/{roomID}/state/{type}", httputil.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], "") })).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", internal.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", httputil.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -183,8 +182,8 @@ func Setup( })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}", - internal.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -199,8 +198,8 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{eventType}/{stateKey}", - internal.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -209,21 +208,21 @@ func Setup( }), ).Methods(http.MethodPut, http.MethodOptions) - r0mux.Handle("/register", internal.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse { + r0mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse { return Register(req, accountDB, deviceDB, cfg) })).Methods(http.MethodPost, http.MethodOptions) - v1mux.Handle("/register", internal.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse { + v1mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse { return LegacyRegister(req, accountDB, deviceDB, cfg) })).Methods(http.MethodPost, http.MethodOptions) - r0mux.Handle("/register/available", internal.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse { + r0mux.Handle("/register/available", httputil.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse { return RegisterAvailable(req, cfg, accountDB) })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/directory/room/{roomAlias}", - internal.MakeExternalAPI("directory_room", func(req *http.Request) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeExternalAPI("directory_room", func(req *http.Request) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -232,8 +231,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/directory/room/{roomAlias}", - internal.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -242,8 +241,8 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/directory/room/{roomAlias}", - internal.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -252,20 +251,20 @@ func Setup( ).Methods(http.MethodDelete, http.MethodOptions) r0mux.Handle("/logout", - internal.MakeAuthAPI("logout", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("logout", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return Logout(req, deviceDB, device) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/logout/all", - internal.MakeAuthAPI("logout", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("logout", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return LogoutAll(req, deviceDB, device) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/typing/{userID}", - internal.MakeAuthAPI("rooms_typing", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("rooms_typing", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -274,8 +273,8 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/sendToDevice/{eventType}/{txnID}", - internal.MakeAuthAPI("send_to_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("send_to_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -288,8 +287,8 @@ func Setup( // rather than r0. It's an exact duplicate of the above handler. // TODO: Remove this if/when sytest is fixed! unstableMux.Handle("/sendToDevice/{eventType}/{txnID}", - internal.MakeAuthAPI("send_to_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("send_to_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -299,7 +298,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/account/whoami", - internal.MakeAuthAPI("whoami", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("whoami", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return Whoami(req, device) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -307,20 +306,20 @@ func Setup( // Stub endpoints required by Riot r0mux.Handle("/login", - internal.MakeExternalAPI("login", func(req *http.Request) util.JSONResponse { + httputil.MakeExternalAPI("login", func(req *http.Request) util.JSONResponse { return Login(req, accountDB, deviceDB, cfg) }), ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) r0mux.Handle("/auth/{authType}/fallback/web", - internal.MakeHTMLAPI("auth_fallback", func(w http.ResponseWriter, req *http.Request) *util.JSONResponse { + httputil.MakeHTMLAPI("auth_fallback", func(w http.ResponseWriter, req *http.Request) *util.JSONResponse { vars := mux.Vars(req) return AuthFallback(w, req, vars["authType"], cfg) }), ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) r0mux.Handle("/pushrules/", - internal.MakeExternalAPI("push_rules", func(req *http.Request) util.JSONResponse { + httputil.MakeExternalAPI("push_rules", func(req *http.Request) util.JSONResponse { // TODO: Implement push rules API res := json.RawMessage(`{ "global": { @@ -339,8 +338,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/user/{userId}/filter", - internal.MakeAuthAPI("put_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("put_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -349,8 +348,8 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/user/{userId}/filter/{filterId}", - internal.MakeAuthAPI("get_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("get_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -361,8 +360,8 @@ func Setup( // Riot user settings r0mux.Handle("/profile/{userID}", - internal.MakeExternalAPI("profile", func(req *http.Request) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeExternalAPI("profile", func(req *http.Request) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -371,8 +370,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/profile/{userID}/avatar_url", - internal.MakeExternalAPI("profile_avatar_url", func(req *http.Request) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeExternalAPI("profile_avatar_url", func(req *http.Request) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -381,8 +380,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/profile/{userID}/avatar_url", - internal.MakeAuthAPI("profile_avatar_url", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("profile_avatar_url", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -393,8 +392,8 @@ func Setup( // PUT requests, so we need to allow this method r0mux.Handle("/profile/{userID}/displayname", - internal.MakeExternalAPI("profile_displayname", func(req *http.Request) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeExternalAPI("profile_displayname", func(req *http.Request) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -403,8 +402,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/profile/{userID}/displayname", - internal.MakeAuthAPI("profile_displayname", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("profile_displayname", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -415,32 +414,32 @@ func Setup( // PUT requests, so we need to allow this method r0mux.Handle("/account/3pid", - internal.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return GetAssociated3PIDs(req, accountDB, device) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/account/3pid", - internal.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return CheckAndSave3PIDAssociation(req, accountDB, device, cfg) }), ).Methods(http.MethodPost, http.MethodOptions) unstableMux.Handle("/account/3pid/delete", - internal.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return Forget3PID(req, accountDB) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/{path:(?:account/3pid|register)}/email/requestToken", - internal.MakeExternalAPI("account_3pid_request_token", func(req *http.Request) util.JSONResponse { + httputil.MakeExternalAPI("account_3pid_request_token", func(req *http.Request) util.JSONResponse { return RequestEmailToken(req, accountDB, cfg) }), ).Methods(http.MethodPost, http.MethodOptions) // Riot logs get flooded unless this is handled r0mux.Handle("/presence/{userID}/status", - internal.MakeExternalAPI("presence", func(req *http.Request) util.JSONResponse { + httputil.MakeExternalAPI("presence", func(req *http.Request) util.JSONResponse { // TODO: Set presence (probably the responsibility of a presence server not clientapi) return util.JSONResponse{ Code: http.StatusOK, @@ -450,13 +449,13 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/voip/turnServer", - internal.MakeAuthAPI("turn_server", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("turn_server", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return RequestTurnServer(req, device, cfg) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/thirdparty/protocols", - internal.MakeExternalAPI("thirdparty_protocols", func(req *http.Request) util.JSONResponse { + httputil.MakeExternalAPI("thirdparty_protocols", func(req *http.Request) util.JSONResponse { // TODO: Return the third party protcols return util.JSONResponse{ Code: http.StatusOK, @@ -466,7 +465,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/initialSync", - internal.MakeExternalAPI("rooms_initial_sync", func(req *http.Request) util.JSONResponse { + httputil.MakeExternalAPI("rooms_initial_sync", func(req *http.Request) util.JSONResponse { // TODO: Allow people to peek into rooms. return util.JSONResponse{ Code: http.StatusForbidden, @@ -476,8 +475,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/user/{userID}/account_data/{type}", - internal.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -486,8 +485,8 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}", - internal.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -496,8 +495,8 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/user/{userID}/account_data/{type}", - internal.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -506,8 +505,8 @@ func Setup( ).Methods(http.MethodGet) r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}", - internal.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -516,8 +515,8 @@ func Setup( ).Methods(http.MethodGet) r0mux.Handle("/rooms/{roomID}/members", - internal.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -526,8 +525,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/joined_members", - internal.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -536,21 +535,21 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/read_markers", - internal.MakeExternalAPI("rooms_read_markers", func(req *http.Request) util.JSONResponse { + httputil.MakeExternalAPI("rooms_read_markers", func(req *http.Request) util.JSONResponse { // TODO: return the read_markers. return util.JSONResponse{Code: http.StatusOK, JSON: struct{}{}} }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/devices", - internal.MakeAuthAPI("get_devices", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("get_devices", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return GetDevicesByLocalpart(req, deviceDB, device) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/devices/{deviceID}", - internal.MakeAuthAPI("get_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("get_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -559,8 +558,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/devices/{deviceID}", - internal.MakeAuthAPI("device_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("device_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -569,8 +568,8 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/devices/{deviceID}", - internal.MakeAuthAPI("delete_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("delete_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -579,14 +578,14 @@ func Setup( ).Methods(http.MethodDelete, http.MethodOptions) r0mux.Handle("/delete_devices", - internal.MakeAuthAPI("delete_devices", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("delete_devices", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return DeleteDevices(req, deviceDB, device) }), ).Methods(http.MethodPost, http.MethodOptions) // Stub implementations for sytest r0mux.Handle("/events", - internal.MakeExternalAPI("events", func(req *http.Request) util.JSONResponse { + httputil.MakeExternalAPI("events", func(req *http.Request) util.JSONResponse { return util.JSONResponse{Code: http.StatusOK, JSON: map[string]interface{}{ "chunk": []interface{}{}, "start": "", @@ -596,7 +595,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/initialSync", - internal.MakeExternalAPI("initial_sync", func(req *http.Request) util.JSONResponse { + httputil.MakeExternalAPI("initial_sync", func(req *http.Request) util.JSONResponse { return util.JSONResponse{Code: http.StatusOK, JSON: map[string]interface{}{ "end": "", }} @@ -604,8 +603,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/user/{userId}/rooms/{roomId}/tags", - internal.MakeAuthAPI("get_tags", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("get_tags", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -614,8 +613,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}", - internal.MakeAuthAPI("put_tag", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("put_tag", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -624,8 +623,8 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}", - internal.MakeAuthAPI("delete_tag", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("delete_tag", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -634,7 +633,7 @@ func Setup( ).Methods(http.MethodDelete, http.MethodOptions) r0mux.Handle("/capabilities", - internal.MakeAuthAPI("capabilities", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("capabilities", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return GetCapabilities(req, rsAPI) }), ).Methods(http.MethodGet) diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index 5d5507e85..77a370778 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -20,8 +20,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/internal/transactions" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -146,8 +146,8 @@ func generateSendEvent( } var queryRes api.QueryLatestEventsAndStateResponse - e, err := internal.BuildEvent(req.Context(), &builder, cfg, evTime, rsAPI, &queryRes) - if err == internal.ErrRoomNoExists { + e, err := eventutil.BuildEvent(req.Context(), &builder, cfg, evTime, rsAPI, &queryRes) + if err == eventutil.ErrRoomNoExists { return nil, &util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound("Room does not exist"), @@ -158,7 +158,7 @@ func generateSendEvent( JSON: jsonerror.BadJSON(e.Error()), } } else if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("internal.BuildEvent failed") + util.GetLogger(req.Context()).WithError(err).Error("eventutil.BuildEvent failed") resErr := jsonerror.InternalServerError() return nil, &resErr } diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index b0df0dd4d..11bf965d4 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -26,8 +26,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" ) @@ -353,7 +353,7 @@ func emit3PIDInviteEvent( } queryRes := api.QueryLatestEventsAndStateResponse{} - event, err := internal.BuildEvent(ctx, builder, cfg, evTime, rsAPI, &queryRes) + event, err := eventutil.BuildEvent(ctx, builder, cfg, evTime, rsAPI, &queryRes) if err != nil { return err } diff --git a/cmd/dendrite-appservice-server/main.go b/cmd/dendrite-appservice-server/main.go index fe00a117d..ec68940af 100644 --- a/cmd/dendrite-appservice-server/main.go +++ b/cmd/dendrite-appservice-server/main.go @@ -16,12 +16,12 @@ package main import ( "github.com/matrix-org/dendrite/appservice" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "AppServiceAPI", true) + cfg := setup.ParseFlags(false) + base := setup.NewBaseDendrite(cfg, "AppServiceAPI", true) defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index 9cf57ef68..a28eb8b3b 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -16,30 +16,27 @@ package main import ( "github.com/matrix-org/dendrite/clientapi" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/internal/transactions" ) func main() { - cfg := basecomponent.ParseFlags(false) + cfg := setup.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "ClientAPI", true) + base := setup.NewBaseDendrite(cfg, "ClientAPI", true) defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() deviceDB := base.CreateDeviceDB() federation := base.CreateFederationClient() - serverKeyAPI := base.ServerKeyAPIClient() - keyRing := serverKeyAPI.KeyRing() - asQuery := base.AppserviceHTTPClient() rsAPI := base.RoomserverHTTPClient() fsAPI := base.FederationSenderHTTPClient() eduInputAPI := base.EDUServerClient() clientapi.AddPublicRoutes( - base.PublicAPIMux, base.Cfg, base.KafkaConsumer, base.KafkaProducer, deviceDB, accountDB, federation, keyRing, + base.PublicAPIMux, base.Cfg, base.KafkaConsumer, base.KafkaProducer, deviceDB, accountDB, federation, rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, ) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index b7cbcdc9c..f215606e8 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -32,8 +32,8 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/federationsender" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/serverkeyapi" @@ -172,7 +172,7 @@ func main() { } monolith.AddAllPublicRoutes(base.Base.PublicAPIMux) - internal.SetupHTTPAPI( + httputil.SetupHTTPAPI( http.DefaultServeMux, base.Base.PublicAPIMux, base.Base.InternalAPIMux, diff --git a/cmd/dendrite-demo-libp2p/p2pdendrite.go b/cmd/dendrite-demo-libp2p/p2pdendrite.go index b7c5c66b0..4270143f5 100644 --- a/cmd/dendrite-demo-libp2p/p2pdendrite.go +++ b/cmd/dendrite-demo-libp2p/p2pdendrite.go @@ -22,7 +22,7 @@ import ( pstore "github.com/libp2p/go-libp2p-core/peerstore" record "github.com/libp2p/go-libp2p-record" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/libp2p/go-libp2p" circuit "github.com/libp2p/go-libp2p-circuit" @@ -39,7 +39,7 @@ import ( // P2PDendrite is a Peer-to-Peer variant of BaseDendrite. type P2PDendrite struct { - Base basecomponent.BaseDendrite + Base setup.BaseDendrite // Store our libp2p object so that we can make outgoing connections from it // later @@ -54,7 +54,7 @@ type P2PDendrite struct { // The componentName is used for logging purposes, and should be a friendly name // of the component running, e.g. SyncAPI. func NewP2PDendrite(cfg *config.Dendrite, componentName string) *P2PDendrite { - baseDendrite := basecomponent.NewBaseDendrite(cfg, componentName, false) + baseDendrite := setup.NewBaseDendrite(cfg, componentName, false) ctx, cancel := context.WithCancel(context.Background()) diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 6ef56d325..e4d4fe9e6 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -34,9 +34,8 @@ import ( "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationsender" - "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" @@ -61,7 +60,7 @@ func (y *yggroundtripper) RoundTrip(req *http.Request) (*http.Response, error) { } func createFederationClient( - base *basecomponent.BaseDendrite, n *yggconn.Node, + base *setup.BaseDendrite, n *yggconn.Node, ) *gomatrixserverlib.FederationClient { yggdialer := func(_, address string) (net.Conn, error) { tokens := strings.Split(address, ":") @@ -135,7 +134,7 @@ func main() { panic(err) } - base := basecomponent.NewBaseDendrite(cfg, "Monolith", false) + base := setup.NewBaseDendrite(cfg, "Monolith", false) defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() @@ -188,7 +187,7 @@ func main() { } monolith.AddAllPublicRoutes(base.PublicAPIMux) - internal.SetupHTTPAPI( + httputil.SetupHTTPAPI( http.DefaultServeMux, base.PublicAPIMux, base.InternalAPIMux, diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/node.go b/cmd/dendrite-demo-yggdrasil/yggconn/node.go index 14bbf84d7..a625f8d8d 100644 --- a/cmd/dendrite-demo-yggdrasil/yggconn/node.go +++ b/cmd/dendrite-demo-yggdrasil/yggconn/node.go @@ -128,6 +128,10 @@ func (n *Node) DerivedServerName() string { return hex.EncodeToString(n.SigningPublicKey()) } +func (n *Node) DerivedSessionName() string { + return hex.EncodeToString(n.EncryptionPublicKey()) +} + func (n *Node) EncryptionPublicKey() []byte { edkey := n.SigningPublicKey() return convert.Ed25519PublicKeyToCurve25519(edkey) diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/session.go b/cmd/dendrite-demo-yggdrasil/yggconn/session.go index 8d96a9cbc..c50b6b73c 100644 --- a/cmd/dendrite-demo-yggdrasil/yggconn/session.go +++ b/cmd/dendrite-demo-yggdrasil/yggconn/session.go @@ -42,7 +42,7 @@ func (n *Node) listenFromYgg() { var session *yamux.Session // If the remote address is lower than ours then we'll be the // server. Otherwse we'll be the client. - if strings.Compare(conn.RemoteAddr().String(), n.DerivedServerName()) < 0 { + if strings.Compare(conn.RemoteAddr().String(), n.DerivedSessionName()) < 0 { session, err = yamux.Server(conn, n.yamuxConfig()) } else { session, err = yamux.Client(conn, n.yamuxConfig()) @@ -105,7 +105,7 @@ func (n *Node) DialContext(ctx context.Context, network, address string) (net.Co } // If the remote address is lower than ours then we will be the // server. Otherwise we'll be the client. - if strings.Compare(conn.RemoteAddr().String(), n.DerivedServerName()) < 0 { + if strings.Compare(conn.RemoteAddr().String(), n.DerivedSessionName()) < 0 { session, err = yamux.Server(conn, n.yamuxConfig()) } else { session, err = yamux.Client(conn, n.yamuxConfig()) diff --git a/cmd/dendrite-edu-server/main.go b/cmd/dendrite-edu-server/main.go index a86f66c89..1ecce884d 100644 --- a/cmd/dendrite-edu-server/main.go +++ b/cmd/dendrite-edu-server/main.go @@ -17,13 +17,13 @@ import ( "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/sirupsen/logrus" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "EDUServerAPI", true) + cfg := setup.ParseFlags(false) + base := setup.NewBaseDendrite(cfg, "EDUServerAPI", true) defer func() { if err := base.Close(); err != nil { logrus.WithError(err).Warn("BaseDendrite close failed") diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index 703d36094..49c2fe0da 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -16,12 +16,12 @@ package main import ( "github.com/matrix-org/dendrite/federationapi" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "FederationAPI", true) + cfg := setup.ParseFlags(false) + base := setup.NewBaseDendrite(cfg, "FederationAPI", true) defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-federation-sender-server/main.go index 7b60ad055..20bc1070f 100644 --- a/cmd/dendrite-federation-sender-server/main.go +++ b/cmd/dendrite-federation-sender-server/main.go @@ -16,12 +16,12 @@ package main import ( "github.com/matrix-org/dendrite/federationsender" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "FederationSender", true) + cfg := setup.ParseFlags(false) + base := setup.NewBaseDendrite(cfg, "FederationSender", true) defer base.Close() // nolint: errcheck federation := base.CreateFederationClient() diff --git a/cmd/dendrite-key-server/main.go b/cmd/dendrite-key-server/main.go index 5dccc3004..06629d39a 100644 --- a/cmd/dendrite-key-server/main.go +++ b/cmd/dendrite-key-server/main.go @@ -15,13 +15,13 @@ package main import ( - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/keyserver" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "KeyServer", true) + cfg := setup.ParseFlags(false) + base := setup.NewBaseDendrite(cfg, "KeyServer", true) defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() diff --git a/cmd/dendrite-media-api-server/main.go b/cmd/dendrite-media-api-server/main.go index 5c65fad05..52c760273 100644 --- a/cmd/dendrite-media-api-server/main.go +++ b/cmd/dendrite-media-api-server/main.go @@ -15,13 +15,13 @@ package main import ( - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/mediaapi" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "MediaAPI", true) + cfg := setup.ParseFlags(false) + base := setup.NewBaseDendrite(cfg, "MediaAPI", true) defer base.Close() // nolint: errcheck deviceDB := base.CreateDeviceDB() diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 195a1ac50..ea8160b84 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -17,17 +17,18 @@ package main import ( "flag" "net/http" + "os" "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationsender" - "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/serverkeyapi" "github.com/sirupsen/logrus" @@ -39,10 +40,11 @@ var ( certFile = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS") keyFile = flag.String("tls-key", "", "The PEM private key to use for TLS") enableHTTPAPIs = flag.Bool("api", false, "Use HTTP APIs instead of short-circuiting (warning: exposes API endpoints!)") + traceInternal = os.Getenv("DENDRITE_TRACE_INTERNAL") == "1" ) func main() { - cfg := basecomponent.ParseFlags(true) + cfg := setup.ParseFlags(true) if *enableHTTPAPIs { // If the HTTP APIs are enabled then we need to update the Listen // statements in the configuration so that we know where to find @@ -56,7 +58,7 @@ func main() { cfg.Listen.ServerKeyAPI = addr } - base := basecomponent.NewBaseDendrite(cfg, "Monolith", *enableHTTPAPIs) + base := setup.NewBaseDendrite(cfg, "Monolith", *enableHTTPAPIs) defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() @@ -72,14 +74,18 @@ func main() { } keyRing := serverKeyAPI.KeyRing() - rsComponent := roomserver.NewInternalAPI( + rsAPI := roomserver.NewInternalAPI( base, keyRing, federation, ) - rsAPI := rsComponent if base.UseHTTPAPIs { roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI) rsAPI = base.RoomserverHTTPClient() } + if traceInternal { + rsAPI = &api.RoomserverInternalAPITrace{ + Impl: rsAPI, + } + } eduInputAPI := eduserver.NewInternalAPI( base, cache.New(), deviceDB, @@ -102,7 +108,7 @@ func main() { federationsender.AddInternalRoutes(base.InternalAPIMux, fsAPI) fsAPI = base.FederationSenderHTTPClient() } - rsComponent.SetFederationSenderAPI(fsAPI) + rsAPI.SetFederationSenderAPI(fsAPI) publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName) if err != nil { @@ -128,7 +134,7 @@ func main() { } monolith.AddAllPublicRoutes(base.PublicAPIMux) - internal.SetupHTTPAPI( + httputil.SetupHTTPAPI( http.DefaultServeMux, base.PublicAPIMux, base.InternalAPIMux, @@ -140,7 +146,7 @@ func main() { go func() { serv := http.Server{ Addr: *httpBindAddr, - WriteTimeout: basecomponent.HTTPServerTimeout, + WriteTimeout: setup.HTTPServerTimeout, } logrus.Info("Listening on ", serv.Addr) @@ -151,7 +157,7 @@ func main() { go func() { serv := http.Server{ Addr: *httpsBindAddr, - WriteTimeout: basecomponent.HTTPServerTimeout, + WriteTimeout: setup.HTTPServerTimeout, } logrus.Info("Listening on ", serv.Addr) diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go index ff7d7958f..3ba45dc6d 100644 --- a/cmd/dendrite-public-rooms-api-server/main.go +++ b/cmd/dendrite-public-rooms-api-server/main.go @@ -15,15 +15,15 @@ package main import ( - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/publicroomsapi" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/sirupsen/logrus" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "PublicRoomsAPI", true) + cfg := setup.ParseFlags(false) + base := setup.NewBaseDendrite(cfg, "PublicRoomsAPI", true) defer base.Close() // nolint: errcheck deviceDB := base.CreateDeviceDB() diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go index a2f1e9415..627a68677 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-room-server/main.go @@ -15,13 +15,13 @@ package main import ( - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/roomserver" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "RoomServerAPI", true) + cfg := setup.ParseFlags(false) + base := setup.NewBaseDendrite(cfg, "RoomServerAPI", true) defer base.Close() // nolint: errcheck federation := base.CreateFederationClient() diff --git a/cmd/dendrite-server-key-api-server/main.go b/cmd/dendrite-server-key-api-server/main.go index b4bfcbffb..9ffaeee31 100644 --- a/cmd/dendrite-server-key-api-server/main.go +++ b/cmd/dendrite-server-key-api-server/main.go @@ -15,13 +15,13 @@ package main import ( - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/serverkeyapi" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "ServerKeyAPI", true) + cfg := setup.ParseFlags(false) + base := setup.NewBaseDendrite(cfg, "ServerKeyAPI", true) defer base.Close() // nolint: errcheck federation := base.CreateFederationClient() diff --git a/cmd/dendrite-sync-api-server/main.go b/cmd/dendrite-sync-api-server/main.go index a5302f74e..41e796802 100644 --- a/cmd/dendrite-sync-api-server/main.go +++ b/cmd/dendrite-sync-api-server/main.go @@ -15,13 +15,13 @@ package main import ( - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/syncapi" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "SyncAPI", true) + cfg := setup.ParseFlags(false) + base := setup.NewBaseDendrite(cfg, "SyncAPI", true) defer base.Close() // nolint: errcheck deviceDB := base.CreateDeviceDB() diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 0512ee5c9..70672f4df 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -26,9 +26,8 @@ import ( "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationsender" - "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" @@ -184,7 +183,7 @@ func main() { if err := cfg.Derive(); err != nil { logrus.Fatalf("Failed to derive values from config: %s", err) } - base := basecomponent.NewBaseDendrite(cfg, "Monolith", false) + base := setup.NewBaseDendrite(cfg, "Monolith", false) defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() @@ -233,7 +232,7 @@ func main() { } monolith.AddAllPublicRoutes(base.PublicAPIMux) - internal.SetupHTTPAPI( + httputil.SetupHTTPAPI( http.DefaultServeMux, base.PublicAPIMux, base.InternalAPIMux, diff --git a/docs/WIRING-Current.md b/docs/WIRING-Current.md new file mode 100644 index 000000000..62450f2f8 --- /dev/null +++ b/docs/WIRING-Current.md @@ -0,0 +1,71 @@ +This document details how various components communicate with each other. There are two kinds of components: + - Public-facing: exposes CS/SS API endpoints and need to be routed to via client-api-proxy or equivalent. + - Internal-only: exposes internal APIs and produces Kafka events. + +## Internal HTTP APIs + +Not everything can be done using Kafka logs. For example, requesting the latest events in a room is much better suited to +a request/response model like HTTP or RPC. Therefore, components can expose "internal APIs" which sit outside of Kafka logs. +Note in Monolith mode these are actually direct function calls and are not serialised HTTP requests. + +``` + Tier 1 Sync PublicRooms FederationAPI ClientAPI MediaAPI +Public Facing | .-----1------` | | | | | | | | | + 2 | .-------3-----------------` | | | `--------|-|-|-|--11--------------------. + | | | .--------4----------------------------------` | | | | + | | | | .---5-----------` | | | | | | + | | | | | .---6----------------------------` | | | + | | | | | | | .-----7----------` | | + | | | | | | 8 | | 10 | + | | | | | | | | `---9----. | | + V V V V V V V V V V V + Tier 2 Roomserver EDUServer FedSender AppService KeyServer ServerKeyAPI +Internal only | `------------------------12----------^ ^ + `------------------------------------------------------------13----------` + + Client ---> Server +``` +- 1 (PublicRooms -> Roomserver): Calculating current auth for changing visibility +- 2 (Sync -> Roomserver): When making backfill requests +- 3 (FedAPI -> Roomserver): Calculating (prev/auth events) and sending new events, processing backfill/state/state_ids requests +- 4 (ClientAPI -> Roomserver): Calculating (prev/auth events) and sending new events, processing /state requests +- 5 (FedAPI -> EDUServer): Sending typing/send-to-device events +- 6 (ClientAPI -> EDUServer): Sending typing/send-to-device events +- 7 (ClientAPI -> FedSender): Handling directory lookups +- 8 (FedAPI -> FedSender): Resetting backoffs when receiving traffic from a server. Querying joined hosts when handling alias lookup requests +- 9 (FedAPI -> AppService): Working out if the client is an appservice user +- 10 (ClientAPI -> AppService): Working out if the client is an appservice user +- 11 (FedAPI -> ServerKeyAPI): Verifying incoming event signatures +- 12 (FedSender -> ServerKeyAPI): Verifying event signatures of responses (e.g from send_join) +- 13 (Roomserver -> ServerKeyAPI): Verifying event signatures of backfilled events + +## Kafka logs + +``` + .----1--------------------------------------------. + V | + Tier 1 Sync PublicRooms FederationAPI ClientAPI MediaAPI +Public Facing ^ ^ ^ ^ + | | | | + 2 | | | + | `-3------------. | + | | | | + | | | | + | .------4------` | | + | | .--------5-----|------------------------------` + | | | | + Tier 2 Roomserver EDUServer FedSender AppService KeyServer ServerKeyAPI +Internal only | | ^ ^ + | `-----6----------` | + `--------------------7--------` + + +Producer ----> Consumer +``` +- 1 (ClientAPI -> Sync): For tracking account data +- 2 (Roomserver -> Sync): For all data to send to clients +- 3 (EDUServer -> Sync): For typing/send-to-device data to send to clients +- 4 (Roomserver -> PublicRooms): For tracking the current room name/topic/joined count/etc. +- 5 (Roomserver -> ClientAPI): For tracking memberships for profile updates. +- 6 (EDUServer -> FedSender): For sending EDUs over federation +- 7 (Roomserver -> FedSender): For sending PDUs over federation, for tracking joined hosts. diff --git a/eduserver/eduserver.go b/eduserver/eduserver.go index c14be3f67..aa65ff239 100644 --- a/eduserver/eduserver.go +++ b/eduserver/eduserver.go @@ -23,7 +23,7 @@ import ( "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/eduserver/input" "github.com/matrix-org/dendrite/eduserver/inthttp" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" ) // AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions @@ -35,7 +35,7 @@ func AddInternalRoutes(internalMux *mux.Router, inputAPI api.EDUServerInputAPI) // NewInternalAPI returns a concerete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( - base *basecomponent.BaseDendrite, + base *setup.BaseDendrite, eduCache *cache.EDUCache, deviceDB devices.Database, ) api.EDUServerInputAPI { diff --git a/eduserver/input/input.go b/eduserver/input/input.go index 0bbf5b844..6eafce42f 100644 --- a/eduserver/input/input.go +++ b/eduserver/input/input.go @@ -97,6 +97,11 @@ func (t *EDUServerInputAPI) sendTypingEvent(ite *api.InputTypingEvent) error { if err != nil { return err } + logrus.WithFields(logrus.Fields{ + "room_id": ite.RoomID, + "user_id": ite.UserID, + "typing": ite.Typing, + }).Infof("Producing to topic '%s'", t.OutputTypingEventTopic) m := &sarama.ProducerMessage{ Topic: string(t.OutputTypingEventTopic), @@ -132,6 +137,11 @@ func (t *EDUServerInputAPI) sendToDeviceEvent(ise *api.InputSendToDeviceEvent) e devices = append(devices, ise.DeviceID) } + logrus.WithFields(logrus.Fields{ + "user_id": ise.UserID, + "num_devices": len(devices), + "type": ise.Type, + }).Infof("Producing to topic '%s'", t.OutputSendToDeviceEventTopic) for _, device := range devices { ote := &api.OutputSendToDeviceEvent{ UserID: ise.UserID, @@ -139,12 +149,6 @@ func (t *EDUServerInputAPI) sendToDeviceEvent(ise *api.InputSendToDeviceEvent) e SendToDeviceEvent: ise.SendToDeviceEvent, } - logrus.WithFields(logrus.Fields{ - "user_id": ise.UserID, - "device_id": ise.DeviceID, - "event_type": ise.Type, - }).Info("handling send-to-device message") - eventJSON, err := json.Marshal(ote) if err != nil { logrus.WithError(err).Error("sendToDevice failed json.Marshal") diff --git a/eduserver/inthttp/client.go b/eduserver/inthttp/client.go index 149e4fb04..7d0bc1603 100644 --- a/eduserver/inthttp/client.go +++ b/eduserver/inthttp/client.go @@ -6,7 +6,7 @@ import ( "net/http" "github.com/matrix-org/dendrite/eduserver/api" - internalHTTP "github.com/matrix-org/dendrite/internal/http" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/opentracing/opentracing-go" ) @@ -39,7 +39,7 @@ func (h *httpEDUServerInputAPI) InputTypingEvent( defer span.Finish() apiURL := h.eduServerURL + EDUServerInputTypingEventPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // InputSendToDeviceEvent implements EDUServerInputAPI @@ -52,5 +52,5 @@ func (h *httpEDUServerInputAPI) InputSendToDeviceEvent( defer span.Finish() apiURL := h.eduServerURL + EDUServerInputSendToDeviceEventPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } diff --git a/eduserver/inthttp/server.go b/eduserver/inthttp/server.go index 6c7d21a4e..e374513a3 100644 --- a/eduserver/inthttp/server.go +++ b/eduserver/inthttp/server.go @@ -6,14 +6,14 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/eduserver/api" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/util" ) // AddRoutes adds the EDUServerInputAPI handlers to the http.ServeMux. func AddRoutes(t api.EDUServerInputAPI, internalAPIMux *mux.Router) { internalAPIMux.Handle(EDUServerInputTypingEventPath, - internal.MakeInternalAPI("inputTypingEvents", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("inputTypingEvents", func(req *http.Request) util.JSONResponse { var request api.InputTypingEventRequest var response api.InputTypingEventResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -26,7 +26,7 @@ func AddRoutes(t api.EDUServerInputAPI, internalAPIMux *mux.Router) { }), ) internalAPIMux.Handle(EDUServerInputSendToDeviceEventPath, - internal.MakeInternalAPI("inputSendToDeviceEvents", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("inputSendToDeviceEvents", func(req *http.Request) util.JSONResponse { var request api.InputSendToDeviceEventRequest var response api.InputSendToDeviceEventResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index 10bc62630..f906c73c9 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -37,7 +37,7 @@ func Backfill( roomID string, cfg *config.Dendrite, ) util.JSONResponse { - var res api.QueryBackfillResponse + var res api.PerformBackfillResponse var eIDs []string var limit string var exists bool @@ -68,7 +68,7 @@ func Backfill( } // Populate the request. - req := api.QueryBackfillRequest{ + req := api.PerformBackfillRequest{ RoomID: roomID, // we don't know who the successors are for these events, which won't // be a problem because we don't use that information when servicing /backfill requests, @@ -87,8 +87,8 @@ func Backfill( } // Query the roomserver. - if err = rsAPI.QueryBackfill(httpReq.Context(), &req, &res); err != nil { - util.GetLogger(httpReq.Context()).WithError(err).Error("query.QueryBackfill failed") + if err = rsAPI.PerformBackfill(httpReq.Context(), &req, &res); err != nil { + util.GetLogger(httpReq.Context()).WithError(err).Error("query.PerformBackfill failed") return jsonerror.InternalServerError() } diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 593aa169f..e01f077a3 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -21,8 +21,8 @@ import ( "time" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -95,8 +95,8 @@ func MakeJoin( queryRes := api.QueryLatestEventsAndStateResponse{ RoomVersion: verRes.RoomVersion, } - event, err := internal.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes) - if err == internal.ErrRoomNoExists { + event, err := eventutil.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes) + if err == eventutil.ErrRoomNoExists { return util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound("Room does not exist"), @@ -107,7 +107,7 @@ func MakeJoin( JSON: jsonerror.BadJSON(e.Error()), } } else if err != nil { - util.GetLogger(httpReq.Context()).WithError(err).Error("internal.BuildEvent failed") + util.GetLogger(httpReq.Context()).WithError(err).Error("eventutil.BuildEvent failed") return jsonerror.InternalServerError() } diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index f998be45a..de15c32ad 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -17,8 +17,8 @@ import ( "time" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -69,8 +69,8 @@ func MakeLeave( } var queryRes api.QueryLatestEventsAndStateResponse - event, err := internal.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes) - if err == internal.ErrRoomNoExists { + event, err := eventutil.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes) + if err == eventutil.ErrRoomNoExists { return util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound("Room does not exist"), @@ -81,7 +81,7 @@ func MakeLeave( JSON: jsonerror.BadJSON(e.Error()), } } else if err != nil { - util.GetLogger(httpReq.Context()).WithError(err).Error("internal.BuildEvent failed") + util.GetLogger(httpReq.Context()).WithError(err).Error("eventutil.BuildEvent failed") return jsonerror.InternalServerError() } diff --git a/federationapi/routing/profile.go b/federationapi/routing/profile.go index 832849845..61d0682bd 100644 --- a/federationapi/routing/profile.go +++ b/federationapi/routing/profile.go @@ -21,8 +21,8 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -72,11 +72,11 @@ func GetProfile( if field != "" { switch field { case "displayname": - res = internal.DisplayName{ + res = eventutil.DisplayName{ DisplayName: profile.DisplayName, } case "avatar_url": - res = internal.AvatarURL{ + res = eventutil.AvatarURL{ AvatarURL: profile.AvatarURL, } default: @@ -84,7 +84,7 @@ func GetProfile( res = jsonerror.InvalidArgumentValue("The request body did not contain an allowed value of argument 'field'. Allowed values are either: 'avatar_url', 'displayname'.") } } else { - res = internal.ProfileResponse{ + res = eventutil.ProfileResponse{ AvatarURL: profile.AvatarURL, DisplayName: profile.DisplayName, } diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 4c5802f2a..92158d3c2 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -23,8 +23,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/httputil" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" serverkeyAPI "github.com/matrix-org/dendrite/serverkeyapi/api" "github.com/matrix-org/gomatrixserverlib" @@ -62,7 +62,7 @@ func Setup( v1fedmux := publicAPIMux.PathPrefix(pathPrefixV1Federation).Subrouter() v2fedmux := publicAPIMux.PathPrefix(pathPrefixV2Federation).Subrouter() - wakeup := &internal.FederationWakeups{ + wakeup := &httputil.FederationWakeups{ FsAPI: fsAPI, } @@ -86,7 +86,7 @@ func Setup( v2keysmux.Handle("/server/", localKeys).Methods(http.MethodGet) v2keysmux.Handle("/server", localKeys).Methods(http.MethodGet) - v1fedmux.Handle("/send/{txnID}", internal.MakeFedAPI( + v1fedmux.Handle("/send/{txnID}", httputil.MakeFedAPI( "federation_send", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return Send( @@ -96,7 +96,7 @@ func Setup( }, )).Methods(http.MethodPut, http.MethodOptions) - v2fedmux.Handle("/invite/{roomID}/{eventID}", internal.MakeFedAPI( + v2fedmux.Handle("/invite/{roomID}/{eventID}", httputil.MakeFedAPI( "federation_invite", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return Invite( @@ -106,13 +106,13 @@ func Setup( }, )).Methods(http.MethodPut, http.MethodOptions) - v1fedmux.Handle("/3pid/onbind", internal.MakeExternalAPI("3pid_onbind", + v1fedmux.Handle("/3pid/onbind", httputil.MakeExternalAPI("3pid_onbind", func(req *http.Request) util.JSONResponse { return CreateInvitesFrom3PIDInvites(req, rsAPI, asAPI, cfg, federation, accountDB) }, )).Methods(http.MethodPost, http.MethodOptions) - v1fedmux.Handle("/exchange_third_party_invite/{roomID}", internal.MakeFedAPI( + v1fedmux.Handle("/exchange_third_party_invite/{roomID}", httputil.MakeFedAPI( "exchange_third_party_invite", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return ExchangeThirdPartyInvite( @@ -121,7 +121,7 @@ func Setup( }, )).Methods(http.MethodPut, http.MethodOptions) - v1fedmux.Handle("/event/{eventID}", internal.MakeFedAPI( + v1fedmux.Handle("/event/{eventID}", httputil.MakeFedAPI( "federation_get_event", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetEvent( @@ -130,7 +130,7 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/state/{roomID}", internal.MakeFedAPI( + v1fedmux.Handle("/state/{roomID}", httputil.MakeFedAPI( "federation_get_state", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetState( @@ -139,7 +139,7 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/state_ids/{roomID}", internal.MakeFedAPI( + v1fedmux.Handle("/state_ids/{roomID}", httputil.MakeFedAPI( "federation_get_state_ids", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetStateIDs( @@ -148,7 +148,7 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/event_auth/{roomID}/{eventID}", internal.MakeFedAPI( + v1fedmux.Handle("/event_auth/{roomID}/{eventID}", httputil.MakeFedAPI( "federation_get_event_auth", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetEventAuth( @@ -157,7 +157,7 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/query/directory", internal.MakeFedAPI( + v1fedmux.Handle("/query/directory", httputil.MakeFedAPI( "federation_query_room_alias", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return RoomAliasToID( @@ -166,7 +166,7 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/query/profile", internal.MakeFedAPI( + v1fedmux.Handle("/query/profile", httputil.MakeFedAPI( "federation_query_profile", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetProfile( @@ -175,7 +175,7 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/user/devices/{userID}", internal.MakeFedAPI( + v1fedmux.Handle("/user/devices/{userID}", httputil.MakeFedAPI( "federation_user_devices", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetUserDevices( @@ -184,7 +184,7 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/make_join/{roomID}/{eventID}", internal.MakeFedAPI( + v1fedmux.Handle("/make_join/{roomID}/{eventID}", httputil.MakeFedAPI( "federation_make_join", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { roomID := vars["roomID"] @@ -209,7 +209,7 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/send_join/{roomID}/{eventID}", internal.MakeFedAPI( + v1fedmux.Handle("/send_join/{roomID}/{eventID}", httputil.MakeFedAPI( "federation_send_join", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { roomID := vars["roomID"] @@ -227,7 +227,7 @@ func Setup( }, )).Methods(http.MethodPut) - v2fedmux.Handle("/send_join/{roomID}/{eventID}", internal.MakeFedAPI( + v2fedmux.Handle("/send_join/{roomID}/{eventID}", httputil.MakeFedAPI( "federation_send_join", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { roomID := vars["roomID"] @@ -238,7 +238,7 @@ func Setup( }, )).Methods(http.MethodPut) - v1fedmux.Handle("/make_leave/{roomID}/{eventID}", internal.MakeFedAPI( + v1fedmux.Handle("/make_leave/{roomID}/{eventID}", httputil.MakeFedAPI( "federation_make_leave", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { roomID := vars["roomID"] @@ -249,7 +249,7 @@ func Setup( }, )).Methods(http.MethodGet) - v2fedmux.Handle("/send_leave/{roomID}/{eventID}", internal.MakeFedAPI( + v2fedmux.Handle("/send_leave/{roomID}/{eventID}", httputil.MakeFedAPI( "federation_send_leave", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { roomID := vars["roomID"] @@ -260,21 +260,21 @@ func Setup( }, )).Methods(http.MethodPut) - v1fedmux.Handle("/version", internal.MakeExternalAPI( + v1fedmux.Handle("/version", httputil.MakeExternalAPI( "federation_version", func(httpReq *http.Request) util.JSONResponse { return Version() }, )).Methods(http.MethodGet) - v1fedmux.Handle("/get_missing_events/{roomID}", internal.MakeFedAPI( + v1fedmux.Handle("/get_missing_events/{roomID}", httputil.MakeFedAPI( "federation_get_missing_events", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetMissingEvents(httpReq, request, rsAPI, vars["roomID"]) }, )).Methods(http.MethodPost) - v1fedmux.Handle("/backfill/{roomID}", internal.MakeFedAPI( + v1fedmux.Handle("/backfill/{roomID}", httputil.MakeFedAPI( "federation_backfill", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return Backfill(httpReq, request, rsAPI, vars["roomID"], cfg) diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index b81f1c003..3123b55c9 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -128,7 +128,6 @@ func (t *testRoomserverAPI) QueryLatestEventsAndState( response *api.QueryLatestEventsAndStateResponse, ) error { r := t.queryLatestEventsAndState(request) - response.QueryLatestEventsAndStateRequest = *request response.RoomExists = r.RoomExists response.RoomVersion = testRoomVersion response.LatestEvents = r.LatestEvents @@ -144,7 +143,6 @@ func (t *testRoomserverAPI) QueryStateAfterEvents( response *api.QueryStateAfterEventsResponse, ) error { response.RoomVersion = testRoomVersion - response.QueryStateAfterEventsRequest = *request res := t.queryStateAfterEvents(request) response.PrevEventsExist = res.PrevEventsExist response.RoomExists = res.RoomExists @@ -181,15 +179,6 @@ func (t *testRoomserverAPI) QueryMembershipsForRoom( return fmt.Errorf("not implemented") } -// Query a list of invite event senders for a user in a room. -func (t *testRoomserverAPI) QueryInvitesForUser( - ctx context.Context, - request *api.QueryInvitesForUserRequest, - response *api.QueryInvitesForUserResponse, -) error { - return fmt.Errorf("not implemented") -} - // Query whether a server is allowed to see an event func (t *testRoomserverAPI) QueryServerAllowedToSeeEvent( ctx context.Context, @@ -220,10 +209,10 @@ func (t *testRoomserverAPI) QueryStateAndAuthChain( } // Query a given amount (or less) of events prior to a given set of events. -func (t *testRoomserverAPI) QueryBackfill( +func (t *testRoomserverAPI) PerformBackfill( ctx context.Context, - request *api.QueryBackfillRequest, - response *api.QueryBackfillResponse, + request *api.PerformBackfillRequest, + response *api.PerformBackfillResponse, ) error { return fmt.Errorf("not implemented") } @@ -621,7 +610,6 @@ func TestTransactionFetchMissingStateByStateIDs(t *testing.T) { } } } - res.QueryEventsByIDRequest = *req return res }, } diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go index f90c494c3..04a18904b 100644 --- a/federationapi/routing/state.go +++ b/federationapi/routing/state.go @@ -107,15 +107,13 @@ func getState( return nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil} } - authEventIDs := getIDsFromEventRef(event.AuthEvents()) - var response api.QueryStateAndAuthChainResponse err := rsAPI.QueryStateAndAuthChain( ctx, &api.QueryStateAndAuthChainRequest{ RoomID: roomID, PrevEventIDs: []string{eventID}, - AuthEventIDs: authEventIDs, + AuthEventIDs: event.AuthEventIDs(), }, &response, ) @@ -134,15 +132,6 @@ func getState( }, nil } -func getIDsFromEventRef(events []gomatrixserverlib.EventReference) []string { - IDs := make([]string, len(events)) - for i := range events { - IDs[i] = events[i].EventID - } - - return IDs -} - func getIDsFromEvent(events []gomatrixserverlib.Event) []string { IDs := make([]string, len(events)) for i := range events { diff --git a/federationsender/api/api.go b/federationsender/api/api.go index d8251b54d..02c762582 100644 --- a/federationsender/api/api.go +++ b/federationsender/api/api.go @@ -15,14 +15,6 @@ type FederationSenderInternalAPI interface { request *PerformDirectoryLookupRequest, response *PerformDirectoryLookupResponse, ) error - // Query the joined hosts and the membership events accounting for their participation in a room. - // Note that if a server has multiple users in the room, it will have multiple entries in the returned slice. - // See `QueryJoinedHostServerNamesInRoom` for a de-duplicated version. - QueryJoinedHostsInRoom( - ctx context.Context, - request *QueryJoinedHostsInRoomRequest, - response *QueryJoinedHostsInRoomResponse, - ) error // Query the server names of the joined hosts in a room. // Unlike QueryJoinedHostsInRoom, this function returns a de-duplicated slice // containing only the server names (without information for membership events). @@ -88,22 +80,12 @@ type PerformServersAliveRequest struct { type PerformServersAliveResponse struct { } -// QueryJoinedHostsInRoomRequest is a request to QueryJoinedHostsInRoom -type QueryJoinedHostsInRoomRequest struct { - RoomID string `json:"room_id"` -} - -// QueryJoinedHostsInRoomResponse is a response to QueryJoinedHostsInRoom -type QueryJoinedHostsInRoomResponse struct { - JoinedHosts []types.JoinedHost `json:"joined_hosts"` -} - -// QueryJoinedHostServerNamesRequest is a request to QueryJoinedHostServerNames +// QueryJoinedHostServerNamesInRoomRequest is a request to QueryJoinedHostServerNames type QueryJoinedHostServerNamesInRoomRequest struct { RoomID string `json:"room_id"` } -// QueryJoinedHostServerNamesResponse is a response to QueryJoinedHostServerNames +// QueryJoinedHostServerNamesInRoomResponse is a response to QueryJoinedHostServerNames type QueryJoinedHostServerNamesInRoomResponse struct { ServerNames []gomatrixserverlib.ServerName `json:"server_names"` } diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index 5f8a555bd..299c7b37a 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -86,11 +86,6 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { switch output.Type { case api.OutputTypeNewRoomEvent: ev := &output.NewRoomEvent.Event - log.WithFields(log.Fields{ - "event_id": ev.EventID(), - "room_id": ev.RoomID(), - "send_as_server": output.NewRoomEvent.SendAsServer, - }).Info("received room event from roomserver") if err := s.processMessage(*output.NewRoomEvent); err != nil { // panic rather than continue with an inconsistent database @@ -131,11 +126,7 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { // processMessage updates the list of currently joined hosts in the room // and then sends the event to the hosts that were joined before the event. func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) error { - addsStateEvents, err := s.lookupStateEvents(ore.AddsStateEventIDs, ore.Event.Event) - if err != nil { - return err - } - addsJoinedHosts, err := joinedHostsFromEvents(addsStateEvents) + addsJoinedHosts, err := joinedHostsFromEvents(gomatrixserverlib.UnwrapEventHeaders(ore.AddsState())) if err != nil { return err } diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index 621920ef5..10ac51c8a 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -23,7 +23,7 @@ import ( "github.com/matrix-org/dendrite/federationsender/queue" "github.com/matrix-org/dendrite/federationsender/storage" "github.com/matrix-org/dendrite/federationsender/types" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" @@ -38,7 +38,7 @@ func AddInternalRoutes(router *mux.Router, intAPI api.FederationSenderInternalAP // NewInternalAPI returns a concerete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( - base *basecomponent.BaseDendrite, + base *setup.BaseDendrite, federation *gomatrixserverlib.FederationClient, rsAPI roomserverAPI.RoomserverInternalAPI, keyRing *gomatrixserverlib.KeyRing, diff --git a/federationsender/internal/query.go b/federationsender/internal/query.go index 88dd50a62..253400a2d 100644 --- a/federationsender/internal/query.go +++ b/federationsender/internal/query.go @@ -7,16 +7,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -// QueryJoinedHostsInRoom implements api.FederationSenderInternalAPI -func (f *FederationSenderInternalAPI) QueryJoinedHostsInRoom( - ctx context.Context, - request *api.QueryJoinedHostsInRoomRequest, - response *api.QueryJoinedHostsInRoomResponse, -) (err error) { - response.JoinedHosts, err = f.db.GetJoinedHosts(ctx, request.RoomID) - return -} - // QueryJoinedHostServerNamesInRoom implements api.FederationSenderInternalAPI func (f *FederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom( ctx context.Context, diff --git a/federationsender/inthttp/client.go b/federationsender/inthttp/client.go index 8b3a106d8..5da4b35f9 100644 --- a/federationsender/inthttp/client.go +++ b/federationsender/inthttp/client.go @@ -6,13 +6,12 @@ import ( "net/http" "github.com/matrix-org/dendrite/federationsender/api" - internalHTTP "github.com/matrix-org/dendrite/internal/http" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/opentracing/opentracing-go" ) // HTTP paths for the internal HTTP API const ( - FederationSenderQueryJoinedHostsInRoomPath = "/federationsender/queryJoinedHostsInRoom" FederationSenderQueryJoinedHostServerNamesInRoomPath = "/federationsender/queryJoinedHostServerNamesInRoom" FederationSenderPerformDirectoryLookupRequestPath = "/federationsender/performDirectoryLookup" @@ -45,7 +44,7 @@ func (h *httpFederationSenderInternalAPI) PerformLeave( defer span.Finish() apiURL := h.federationSenderURL + FederationSenderPerformLeaveRequestPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } func (h *httpFederationSenderInternalAPI) PerformServersAlive( @@ -57,7 +56,7 @@ func (h *httpFederationSenderInternalAPI) PerformServersAlive( defer span.Finish() apiURL := h.federationSenderURL + FederationSenderPerformServersAlivePath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryJoinedHostServerNamesInRoom implements FederationSenderInternalAPI @@ -70,20 +69,7 @@ func (h *httpFederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom( defer span.Finish() apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostServerNamesInRoomPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// QueryJoinedHostsInRoom implements FederationSenderInternalAPI -func (h *httpFederationSenderInternalAPI) QueryJoinedHostsInRoom( - ctx context.Context, - request *api.QueryJoinedHostsInRoomRequest, - response *api.QueryJoinedHostsInRoomResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostsInRoom") - defer span.Finish() - - apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostsInRoomPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // Handle an instruction to make_join & send_join with a remote server. @@ -96,7 +82,7 @@ func (h *httpFederationSenderInternalAPI) PerformJoin( defer span.Finish() apiURL := h.federationSenderURL + FederationSenderPerformJoinRequestPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // Handle an instruction to make_join & send_join with a remote server. @@ -109,5 +95,5 @@ func (h *httpFederationSenderInternalAPI) PerformDirectoryLookup( defer span.Finish() apiURL := h.federationSenderURL + FederationSenderPerformDirectoryLookupRequestPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } diff --git a/federationsender/inthttp/server.go b/federationsender/inthttp/server.go index c94a14295..babd3ae13 100644 --- a/federationsender/inthttp/server.go +++ b/federationsender/inthttp/server.go @@ -6,29 +6,15 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/federationsender/api" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/util" ) // AddRoutes adds the FederationSenderInternalAPI handlers to the http.ServeMux. func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Router) { - internalAPIMux.Handle( - FederationSenderQueryJoinedHostsInRoomPath, - internal.MakeInternalAPI("QueryJoinedHostsInRoom", func(req *http.Request) util.JSONResponse { - var request api.QueryJoinedHostsInRoomRequest - var response api.QueryJoinedHostsInRoomResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := intAPI.QueryJoinedHostsInRoom(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) internalAPIMux.Handle( FederationSenderQueryJoinedHostServerNamesInRoomPath, - internal.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse { var request api.QueryJoinedHostServerNamesInRoomRequest var response api.QueryJoinedHostServerNamesInRoomResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -41,7 +27,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route }), ) internalAPIMux.Handle(FederationSenderPerformJoinRequestPath, - internal.MakeInternalAPI("PerformJoinRequest", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("PerformJoinRequest", func(req *http.Request) util.JSONResponse { var request api.PerformJoinRequest var response api.PerformJoinResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -54,7 +40,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route }), ) internalAPIMux.Handle(FederationSenderPerformLeaveRequestPath, - internal.MakeInternalAPI("PerformLeaveRequest", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("PerformLeaveRequest", func(req *http.Request) util.JSONResponse { var request api.PerformLeaveRequest var response api.PerformLeaveResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -67,7 +53,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route }), ) internalAPIMux.Handle(FederationSenderPerformDirectoryLookupRequestPath, - internal.MakeInternalAPI("PerformDirectoryLookupRequest", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("PerformDirectoryLookupRequest", func(req *http.Request) util.JSONResponse { var request api.PerformDirectoryLookupRequest var response api.PerformDirectoryLookupResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -80,7 +66,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route }), ) internalAPIMux.Handle(FederationSenderPerformServersAlivePath, - internal.MakeInternalAPI("PerformServersAliveRequest", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("PerformServersAliveRequest", func(req *http.Request) util.JSONResponse { var request api.PerformServersAliveRequest var response api.PerformServersAliveResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { diff --git a/federationsender/queue/queue.go b/federationsender/queue/queue.go index 5b8fc3c56..240343559 100644 --- a/federationsender/queue/queue.go +++ b/federationsender/queue/queue.go @@ -107,6 +107,9 @@ func (oqs *OutgoingQueues) SendEvent( // Remove our own server from the list of destinations. destinations = filterAndDedupeDests(oqs.origin, destinations) + if len(destinations) == 0 { + return nil + } log.WithFields(log.Fields{ "destinations": destinations, "event": ev.EventID(), diff --git a/federationsender/storage/postgres/joined_hosts_table.go b/federationsender/storage/postgres/joined_hosts_table.go index 1b898f485..c0f9a7d5b 100644 --- a/federationsender/storage/postgres/joined_hosts_table.go +++ b/federationsender/storage/postgres/joined_hosts_table.go @@ -22,6 +22,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/federationsender/types" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -85,7 +86,7 @@ func (s *joinedHostsStatements) insertJoinedHosts( roomID, eventID string, serverName gomatrixserverlib.ServerName, ) error { - stmt := internal.TxStmt(txn, s.insertJoinedHostsStmt) + stmt := sqlutil.TxStmt(txn, s.insertJoinedHostsStmt) _, err := stmt.ExecContext(ctx, roomID, eventID, serverName) return err } @@ -93,7 +94,7 @@ func (s *joinedHostsStatements) insertJoinedHosts( func (s *joinedHostsStatements) deleteJoinedHosts( ctx context.Context, txn *sql.Tx, eventIDs []string, ) error { - stmt := internal.TxStmt(txn, s.deleteJoinedHostsStmt) + stmt := sqlutil.TxStmt(txn, s.deleteJoinedHostsStmt) _, err := stmt.ExecContext(ctx, pq.StringArray(eventIDs)) return err } @@ -101,7 +102,7 @@ func (s *joinedHostsStatements) deleteJoinedHosts( func (s *joinedHostsStatements) selectJoinedHostsWithTx( ctx context.Context, txn *sql.Tx, roomID string, ) ([]types.JoinedHost, error) { - stmt := internal.TxStmt(txn, s.selectJoinedHostsStmt) + stmt := sqlutil.TxStmt(txn, s.selectJoinedHostsStmt) return joinedHostsFromStmt(ctx, stmt, roomID) } diff --git a/federationsender/storage/postgres/room_table.go b/federationsender/storage/postgres/room_table.go index c945475ba..e5266c635 100644 --- a/federationsender/storage/postgres/room_table.go +++ b/federationsender/storage/postgres/room_table.go @@ -19,7 +19,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" ) const roomSchema = ` @@ -71,7 +71,7 @@ func (s *roomStatements) prepare(db *sql.DB) (err error) { func (s *roomStatements) insertRoom( ctx context.Context, txn *sql.Tx, roomID string, ) error { - _, err := internal.TxStmt(txn, s.insertRoomStmt).ExecContext(ctx, roomID) + _, err := sqlutil.TxStmt(txn, s.insertRoomStmt).ExecContext(ctx, roomID) return err } @@ -82,7 +82,7 @@ func (s *roomStatements) selectRoomForUpdate( ctx context.Context, txn *sql.Tx, roomID string, ) (string, error) { var lastEventID string - stmt := internal.TxStmt(txn, s.selectRoomForUpdateStmt) + stmt := sqlutil.TxStmt(txn, s.selectRoomForUpdateStmt) err := stmt.QueryRowContext(ctx, roomID).Scan(&lastEventID) if err != nil { return "", err @@ -95,7 +95,7 @@ func (s *roomStatements) selectRoomForUpdate( func (s *roomStatements) updateRoom( ctx context.Context, txn *sql.Tx, roomID, lastEventID string, ) error { - stmt := internal.TxStmt(txn, s.updateRoomStmt) + stmt := sqlutil.TxStmt(txn, s.updateRoomStmt) _, err := stmt.ExecContext(ctx, roomID, lastEventID) return err } diff --git a/federationsender/storage/postgres/storage.go b/federationsender/storage/postgres/storage.go index a4733a20c..8fd4c11a3 100644 --- a/federationsender/storage/postgres/storage.go +++ b/federationsender/storage/postgres/storage.go @@ -20,7 +20,6 @@ import ( "database/sql" "github.com/matrix-org/dendrite/federationsender/types" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" ) @@ -28,12 +27,12 @@ import ( type Database struct { joinedHostsStatements roomStatements - internal.PartitionOffsetStatements + sqlutil.PartitionOffsetStatements db *sql.DB } // NewDatabase opens a new database -func NewDatabase(dataSourceName string, dbProperties internal.DbProperties) (*Database, error) { +func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database, error) { var result Database var err error if result.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { @@ -70,7 +69,7 @@ func (d *Database) UpdateRoom( addHosts []types.JoinedHost, removeHosts []string, ) (joinedHosts []types.JoinedHost, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { err = d.insertRoom(ctx, txn, roomID) if err != nil { return err diff --git a/federationsender/storage/sqlite3/joined_hosts_table.go b/federationsender/storage/sqlite3/joined_hosts_table.go index 795d33208..d9824658c 100644 --- a/federationsender/storage/sqlite3/joined_hosts_table.go +++ b/federationsender/storage/sqlite3/joined_hosts_table.go @@ -21,6 +21,7 @@ import ( "github.com/matrix-org/dendrite/federationsender/types" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -84,7 +85,7 @@ func (s *joinedHostsStatements) insertJoinedHosts( roomID, eventID string, serverName gomatrixserverlib.ServerName, ) error { - stmt := internal.TxStmt(txn, s.insertJoinedHostsStmt) + stmt := sqlutil.TxStmt(txn, s.insertJoinedHostsStmt) _, err := stmt.ExecContext(ctx, roomID, eventID, serverName) return err } @@ -93,7 +94,7 @@ func (s *joinedHostsStatements) deleteJoinedHosts( ctx context.Context, txn *sql.Tx, eventIDs []string, ) error { for _, eventID := range eventIDs { - stmt := internal.TxStmt(txn, s.deleteJoinedHostsStmt) + stmt := sqlutil.TxStmt(txn, s.deleteJoinedHostsStmt) if _, err := stmt.ExecContext(ctx, eventID); err != nil { return err } @@ -104,7 +105,7 @@ func (s *joinedHostsStatements) deleteJoinedHosts( func (s *joinedHostsStatements) selectJoinedHostsWithTx( ctx context.Context, txn *sql.Tx, roomID string, ) ([]types.JoinedHost, error) { - stmt := internal.TxStmt(txn, s.selectJoinedHostsStmt) + stmt := sqlutil.TxStmt(txn, s.selectJoinedHostsStmt) return joinedHostsFromStmt(ctx, stmt, roomID) } diff --git a/federationsender/storage/sqlite3/room_table.go b/federationsender/storage/sqlite3/room_table.go index 4bf9ab81d..ca0c4d0b6 100644 --- a/federationsender/storage/sqlite3/room_table.go +++ b/federationsender/storage/sqlite3/room_table.go @@ -19,7 +19,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" ) const roomSchema = ` @@ -71,7 +71,7 @@ func (s *roomStatements) prepare(db *sql.DB) (err error) { func (s *roomStatements) insertRoom( ctx context.Context, txn *sql.Tx, roomID string, ) error { - _, err := internal.TxStmt(txn, s.insertRoomStmt).ExecContext(ctx, roomID) + _, err := sqlutil.TxStmt(txn, s.insertRoomStmt).ExecContext(ctx, roomID) return err } @@ -82,7 +82,7 @@ func (s *roomStatements) selectRoomForUpdate( ctx context.Context, txn *sql.Tx, roomID string, ) (string, error) { var lastEventID string - stmt := internal.TxStmt(txn, s.selectRoomForUpdateStmt) + stmt := sqlutil.TxStmt(txn, s.selectRoomForUpdateStmt) err := stmt.QueryRowContext(ctx, roomID).Scan(&lastEventID) if err != nil { return "", err @@ -95,7 +95,7 @@ func (s *roomStatements) selectRoomForUpdate( func (s *roomStatements) updateRoom( ctx context.Context, txn *sql.Tx, roomID, lastEventID string, ) error { - stmt := internal.TxStmt(txn, s.updateRoomStmt) + stmt := sqlutil.TxStmt(txn, s.updateRoomStmt) _, err := stmt.ExecContext(ctx, roomID, lastEventID) return err } diff --git a/federationsender/storage/sqlite3/storage.go b/federationsender/storage/sqlite3/storage.go index 8699fc19a..ac303f646 100644 --- a/federationsender/storage/sqlite3/storage.go +++ b/federationsender/storage/sqlite3/storage.go @@ -22,7 +22,6 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/matrix-org/dendrite/federationsender/types" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" ) @@ -30,7 +29,7 @@ import ( type Database struct { joinedHostsStatements roomStatements - internal.PartitionOffsetStatements + sqlutil.PartitionOffsetStatements db *sql.DB } @@ -42,7 +41,7 @@ func NewDatabase(dataSourceName string) (*Database, error) { if err != nil { return nil, err } - if result.db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil { + if result.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { return nil, err } if err = result.prepare(); err != nil { @@ -76,7 +75,7 @@ func (d *Database) UpdateRoom( addHosts []types.JoinedHost, removeHosts []string, ) (joinedHosts []types.JoinedHost, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { err = d.insertRoom(ctx, txn, roomID) if err != nil { return err diff --git a/federationsender/storage/storage.go b/federationsender/storage/storage.go index df8433eba..d37360056 100644 --- a/federationsender/storage/storage.go +++ b/federationsender/storage/storage.go @@ -21,11 +21,11 @@ import ( "github.com/matrix-org/dendrite/federationsender/storage/postgres" "github.com/matrix-org/dendrite/federationsender/storage/sqlite3" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" ) // NewDatabase opens a new database -func NewDatabase(dataSourceName string, dbProperties internal.DbProperties) (Database, error) { +func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return postgres.NewDatabase(dataSourceName, dbProperties) diff --git a/federationsender/storage/storage_wasm.go b/federationsender/storage/storage_wasm.go index f593fd448..e5c8f293b 100644 --- a/federationsender/storage/storage_wasm.go +++ b/federationsender/storage/storage_wasm.go @@ -19,13 +19,13 @@ import ( "net/url" "github.com/matrix-org/dendrite/federationsender/storage/sqlite3" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" ) // NewDatabase opens a new database func NewDatabase( dataSourceName string, - dbProperties internal.DbProperties, // nolint:unparam + dbProperties sqlutil.DbProperties, // nolint:unparam ) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { diff --git a/go.mod b/go.mod index a05bb71e2..2f7874082 100644 --- a/go.mod +++ b/go.mod @@ -38,6 +38,7 @@ require ( github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b go.uber.org/atomic v1.4.0 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d + golang.org/x/tools v0.0.0-20200612022331-742c5eb664c2 // indirect gopkg.in/h2non/bimg.v1 v1.0.18 gopkg.in/yaml.v2 v2.2.8 ) diff --git a/go.sum b/go.sum index 00d25a3f6..301066b90 100644 --- a/go.sum +++ b/go.sum @@ -570,6 +570,7 @@ github.com/yggdrasil-network/yggdrasil-go v0.3.14 h1:vWzYzCQxOruS+J5FkLfXOS0JhCJ github.com/yggdrasil-network/yggdrasil-go v0.3.14/go.mod h1:rkQzLzVHlFdzsEMG+bDdTI+KeWPCZq1HpXRFzwinf6M= github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b h1:ELOisSxFXCcptRs4LFub+Hz5fYUvV12wZrTps99Eb3E= github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b/go.mod h1:d+Nz6SPeG6kmeSPFL0cvfWfgwEql75fUnZiAONgvyBE= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -603,6 +604,8 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -670,7 +673,11 @@ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200612022331-742c5eb664c2 h1:DVqHa33CzfnTKwUV6be+I4hp31W6iXn3ZiEcdKGzLyI= +golang.org/x/tools v0.0.0-20200612022331-742c5eb664c2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= diff --git a/internal/caching/cache_serverkeys.go b/internal/caching/cache_serverkeys.go index 77786286a..ac5ba11bf 100644 --- a/internal/caching/cache_serverkeys.go +++ b/internal/caching/cache_serverkeys.go @@ -2,6 +2,7 @@ package caching import ( "fmt" + "time" "github.com/matrix-org/gomatrixserverlib" ) @@ -24,6 +25,7 @@ func (c Caches) GetServerKey( timestamp gomatrixserverlib.Timestamp, ) (gomatrixserverlib.PublicKeyLookupResult, bool) { key := fmt.Sprintf("%s/%s", request.ServerName, request.KeyID) + now := gomatrixserverlib.AsTimestamp(time.Now()) val, found := c.ServerKeys.Get(key) if found && val != nil { if keyLookupResult, ok := val.(gomatrixserverlib.PublicKeyLookupResult); ok { diff --git a/internal/consumers.go b/internal/consumers.go index df68cbfaa..d7917f235 100644 --- a/internal/consumers.go +++ b/internal/consumers.go @@ -19,20 +19,13 @@ import ( "fmt" "github.com/Shopify/sarama" + "github.com/matrix-org/dendrite/internal/sqlutil" ) -// A PartitionOffset is the offset into a partition of the input log. -type PartitionOffset struct { - // The ID of the partition. - Partition int32 - // The offset into the partition. - Offset int64 -} - // A PartitionStorer has the storage APIs needed by the consumer. type PartitionStorer interface { // PartitionOffsets returns the offsets the consumer has reached for each partition. - PartitionOffsets(ctx context.Context, topic string) ([]PartitionOffset, error) + PartitionOffsets(ctx context.Context, topic string) ([]sqlutil.PartitionOffset, error) // SetPartitionOffset records where the consumer has reached for a partition. SetPartitionOffset(ctx context.Context, topic string, partition int32, offset int64) error } diff --git a/internal/eventcontent.go b/internal/eventutil/eventcontent.go similarity index 97% rename from internal/eventcontent.go rename to internal/eventutil/eventcontent.go index 645128369..873e20a8e 100644 --- a/internal/eventcontent.go +++ b/internal/eventutil/eventcontent.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package eventutil import "github.com/matrix-org/gomatrixserverlib" diff --git a/internal/events.go b/internal/eventutil/events.go similarity index 98% rename from internal/events.go rename to internal/eventutil/events.go index 89c82e03b..e6c7a4ff7 100644 --- a/internal/events.go +++ b/internal/eventutil/events.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package eventutil import ( "context" diff --git a/internal/types.go b/internal/eventutil/types.go similarity index 96% rename from internal/types.go rename to internal/eventutil/types.go index be2717f39..6d119ce6d 100644 --- a/internal/types.go +++ b/internal/eventutil/types.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package eventutil import ( "errors" diff --git a/internal/httpapis/paths.go b/internal/httpapis/paths.go deleted file mode 100644 index 8adec2dff..000000000 --- a/internal/httpapis/paths.go +++ /dev/null @@ -1,6 +0,0 @@ -package httpapis - -const ( - PublicPathPrefix = "/_matrix/" - InternalPathPrefix = "/api/" -) diff --git a/internal/http/http.go b/internal/httputil/http.go similarity index 69% rename from internal/http/http.go rename to internal/httputil/http.go index 2b1891403..9197371aa 100644 --- a/internal/http/http.go +++ b/internal/httputil/http.go @@ -1,4 +1,18 @@ -package http +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httputil import ( "bytes" @@ -9,7 +23,6 @@ import ( "net/url" "strings" - "github.com/matrix-org/dendrite/internal/httpapis" opentracing "github.com/opentracing/opentracing-go" "github.com/opentracing/opentracing-go/ext" ) @@ -29,7 +42,7 @@ func PostJSON( return err } - parsedAPIURL.Path = httpapis.InternalPathPrefix + strings.TrimLeft(parsedAPIURL.Path, "/") + parsedAPIURL.Path = InternalPathPrefix + strings.TrimLeft(parsedAPIURL.Path, "/") apiURL = parsedAPIURL.String() req, err := http.NewRequest(http.MethodPost, apiURL, bytes.NewReader(jsonBytes)) diff --git a/internal/httpapi.go b/internal/httputil/httpapi.go similarity index 92% rename from internal/httpapi.go rename to internal/httputil/httpapi.go index 991a98619..0a37f06c2 100644 --- a/internal/httpapi.go +++ b/internal/httputil/httpapi.go @@ -1,4 +1,18 @@ -package internal +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httputil import ( "context" @@ -16,7 +30,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" federationsenderAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal/config" - "github.com/matrix-org/dendrite/internal/httpapis" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" opentracing "github.com/opentracing/opentracing-go" @@ -227,9 +240,9 @@ func SetupHTTPAPI(servMux *http.ServeMux, publicApiMux *mux.Router, internalApiM servMux.Handle("/metrics", WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Metrics.BasicAuth)) } if enableHTTPAPIs { - servMux.Handle(httpapis.InternalPathPrefix, internalApiMux) + servMux.Handle(InternalPathPrefix, internalApiMux) } - servMux.Handle(httpapis.PublicPathPrefix, WrapHandlerInCORS(publicApiMux)) + servMux.Handle(PublicPathPrefix, WrapHandlerInCORS(publicApiMux)) } // WrapHandlerInBasicAuth adds basic auth to a handler. Only used for /metrics diff --git a/internal/httpapi_test.go b/internal/httputil/httpapi_test.go similarity index 75% rename from internal/httpapi_test.go rename to internal/httputil/httpapi_test.go index 6f159c8d6..de6ccf9b4 100644 --- a/internal/httpapi_test.go +++ b/internal/httputil/httpapi_test.go @@ -1,4 +1,18 @@ -package internal +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httputil import ( "net/http" diff --git a/internal/httputil/paths.go b/internal/httputil/paths.go new file mode 100644 index 000000000..728b5a871 --- /dev/null +++ b/internal/httputil/paths.go @@ -0,0 +1,20 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httputil + +const ( + PublicPathPrefix = "/_matrix/" + InternalPathPrefix = "/api/" +) diff --git a/internal/routing.go b/internal/httputil/routing.go similarity index 94% rename from internal/routing.go rename to internal/httputil/routing.go index 4462c70ca..0bd3655ec 100644 --- a/internal/routing.go +++ b/internal/httputil/routing.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Matrix.org Foundation C.I.C. +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package httputil import ( "net/url" diff --git a/internal/basecomponent/base.go b/internal/setup/base.go similarity index 96% rename from internal/basecomponent/base.go rename to internal/setup/base.go index 4b80e3ba8..a6e023b97 100644 --- a/internal/basecomponent/base.go +++ b/internal/setup/base.go @@ -1,4 +1,4 @@ -// Copyright 2017 New Vector Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package basecomponent +package setup import ( "database/sql" @@ -23,7 +23,7 @@ import ( "time" "github.com/matrix-org/dendrite/internal/caching" - "github.com/matrix-org/dendrite/internal/httpapis" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/naffka" @@ -127,8 +127,8 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo tracerCloser: closer, Cfg: cfg, Caches: cache, - PublicAPIMux: httpmux.PathPrefix(httpapis.PublicPathPrefix).Subrouter().UseEncodedPath(), - InternalAPIMux: httpmux.PathPrefix(httpapis.InternalPathPrefix).Subrouter().UseEncodedPath(), + PublicAPIMux: httpmux.PathPrefix(httputil.PublicPathPrefix).Subrouter().UseEncodedPath(), + InternalAPIMux: httpmux.PathPrefix(httputil.InternalPathPrefix).Subrouter().UseEncodedPath(), httpClient: &client, KafkaConsumer: kafkaConsumer, KafkaProducer: kafkaProducer, @@ -237,7 +237,7 @@ func (b *BaseDendrite) SetupAndServeHTTP(bindaddr string, listenaddr string) { WriteTimeout: HTTPServerTimeout, } - internal.SetupHTTPAPI( + httputil.SetupHTTPAPI( http.DefaultServeMux, b.PublicAPIMux, b.InternalAPIMux, @@ -282,7 +282,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { if err != nil { logrus.WithError(err).Panic("Failed to parse naffka database file URI") } - db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil) + db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil) if err != nil { logrus.WithError(err).Panic("Failed to open naffka database") } diff --git a/internal/basecomponent/flags.go b/internal/setup/flags.go similarity index 94% rename from internal/basecomponent/flags.go rename to internal/setup/flags.go index 117df0796..e4fc58d60 100644 --- a/internal/basecomponent/flags.go +++ b/internal/setup/flags.go @@ -1,4 +1,4 @@ -// Copyright 2017 New Vector Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package basecomponent +package setup import ( "flag" diff --git a/internal/setup/monolith.go b/internal/setup/monolith.go index ddda181af..1bf97a15f 100644 --- a/internal/setup/monolith.go +++ b/internal/setup/monolith.go @@ -1,3 +1,17 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package setup import ( @@ -53,7 +67,7 @@ type Monolith struct { func (m *Monolith) AddAllPublicRoutes(publicMux *mux.Router) { clientapi.AddPublicRoutes( publicMux, m.Config, m.KafkaConsumer, m.KafkaProducer, m.DeviceDB, m.AccountDB, - m.FedClient, m.KeyRing, m.RoomserverAPI, + m.FedClient, m.RoomserverAPI, m.EDUInternalAPI, m.AppserviceAPI, transactions.New(), m.FederationSenderAPI, ) diff --git a/internal/partition_offset_table.go b/internal/sqlutil/partition_offset_table.go similarity index 88% rename from internal/partition_offset_table.go rename to internal/sqlutil/partition_offset_table.go index 8b72819b7..348829025 100644 --- a/internal/partition_offset_table.go +++ b/internal/sqlutil/partition_offset_table.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,14 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package sqlutil import ( "context" "database/sql" "strings" + + "github.com/matrix-org/util" ) +// A PartitionOffset is the offset into a partition of the input log. +type PartitionOffset struct { + // The ID of the partition. + Partition int32 + // The offset into the partition. + Offset int64 +} + const partitionOffsetsSchema = ` -- The offsets that the server has processed up to. CREATE TABLE IF NOT EXISTS ${prefix}_partition_offsets ( @@ -90,7 +100,12 @@ func (s *PartitionOffsetStatements) selectPartitionOffsets( if err != nil { return nil, err } - defer CloseAndLogIfError(ctx, rows, "selectPartitionOffsets: rows.close() failed") + defer func() { + err2 := rows.Close() + if err2 != nil { + util.GetLogger(ctx).WithError(err2).Error("selectPartitionOffsets: rows.close() failed") + } + }() var results []PartitionOffset for rows.Next() { var offset PartitionOffset diff --git a/internal/postgres.go b/internal/sqlutil/postgres.go similarity index 98% rename from internal/postgres.go rename to internal/sqlutil/postgres.go index 7ae40d8fd..41a5508a1 100644 --- a/internal/postgres.go +++ b/internal/sqlutil/postgres.go @@ -14,7 +14,7 @@ // +build !wasm -package internal +package sqlutil import "github.com/lib/pq" diff --git a/internal/postgres_wasm.go b/internal/sqlutil/postgres_wasm.go similarity index 97% rename from internal/postgres_wasm.go rename to internal/sqlutil/postgres_wasm.go index 64d328291..c45842f0c 100644 --- a/internal/postgres_wasm.go +++ b/internal/sqlutil/postgres_wasm.go @@ -14,7 +14,7 @@ // +build wasm -package internal +package sqlutil // IsUniqueConstraintViolationErr no-ops for this architecture func IsUniqueConstraintViolationErr(err error) bool { diff --git a/internal/sql.go b/internal/sqlutil/sql.go similarity index 96% rename from internal/sql.go rename to internal/sqlutil/sql.go index e3c10afca..a25a4a5b6 100644 --- a/internal/sql.go +++ b/internal/sqlutil/sql.go @@ -1,6 +1,4 @@ -// Copyright 2017 Vector Creations Ltd -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package sqlutil import ( "database/sql" diff --git a/internal/sqlutil/trace.go b/internal/sqlutil/trace.go index 1b008e1b7..f6644d591 100644 --- a/internal/sqlutil/trace.go +++ b/internal/sqlutil/trace.go @@ -25,7 +25,6 @@ import ( "strings" "time" - "github.com/matrix-org/dendrite/internal" "github.com/ngrok/sqlmw" "github.com/sirupsen/logrus" ) @@ -78,7 +77,7 @@ func (in *traceInterceptor) RowsNext(c context.Context, rows driver.Rows, dest [ // Open opens a database specified by its database driver name and a driver-specific data source name, // usually consisting of at least a database name and connection information. Includes tracing driver // if DENDRITE_TRACE_SQL=1 -func Open(driverName, dsn string, dbProperties internal.DbProperties) (*sql.DB, error) { +func Open(driverName, dsn string, dbProperties DbProperties) (*sql.DB, error) { if tracingEnabled { // install the wrapped driver driverName += "-trace" @@ -87,7 +86,7 @@ func Open(driverName, dsn string, dbProperties internal.DbProperties) (*sql.DB, if err != nil { return nil, err } - if driverName != internal.SQLiteDriverName() && dbProperties != nil { + if driverName != SQLiteDriverName() && dbProperties != nil { logrus.WithFields(logrus.Fields{ "MaxOpenConns": dbProperties.MaxOpenConns(), "MaxIdleConns": dbProperties.MaxIdleConns(), diff --git a/internal/sqlutil/uri.go b/internal/sqlutil/uri.go index f72e02426..703258e6c 100644 --- a/internal/sqlutil/uri.go +++ b/internal/sqlutil/uri.go @@ -1,3 +1,17 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sqlutil import ( diff --git a/keyserver/routing/routing.go b/keyserver/routing/routing.go index 2d1122c62..bce3c46ba 100644 --- a/keyserver/routing/routing.go +++ b/keyserver/routing/routing.go @@ -22,8 +22,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/util" ) @@ -49,7 +49,7 @@ func Setup( } r0mux.Handle("/keys/query", - internal.MakeAuthAPI("queryKeys", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("queryKeys", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return QueryKeys(req) }), ).Methods(http.MethodPost, http.MethodOptions) diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index 11e4ecad5..6bcd3f552 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -22,8 +22,8 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -59,7 +59,7 @@ func Setup( } // TODO: Add AS support - r0mux.Handle("/upload", internal.MakeAuthAPI( + r0mux.Handle("/upload", httputil.MakeAuthAPI( "upload", authData, func(req *http.Request, _ *authtypes.Device) util.JSONResponse { return Upload(req, cfg, db, activeThumbnailGeneration) @@ -99,7 +99,7 @@ func makeDownloadAPI( util.SetCORSHeaders(w) // Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors w.Header().Set("Content-Type", "application/json") - vars, _ := internal.URLDecodeMapValues(mux.Vars(req)) + vars, _ := httputil.URLDecodeMapValues(mux.Vars(req)) Download( w, req, diff --git a/mediaapi/storage/postgres/storage.go b/mediaapi/storage/postgres/storage.go index 72ee05cd8..e45e08416 100644 --- a/mediaapi/storage/postgres/storage.go +++ b/mediaapi/storage/postgres/storage.go @@ -21,7 +21,6 @@ import ( // Import the postgres database driver. _ "github.com/lib/pq" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -34,7 +33,7 @@ type Database struct { } // Open opens a postgres database. -func Open(dataSourceName string, dbProperties internal.DbProperties) (*Database, error) { +func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database, error) { var d Database var err error if d.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { diff --git a/mediaapi/storage/postgres/thumbnail_table.go b/mediaapi/storage/postgres/thumbnail_table.go index dbc97cf22..3f28cdbbf 100644 --- a/mediaapi/storage/postgres/thumbnail_table.go +++ b/mediaapi/storage/postgres/thumbnail_table.go @@ -21,7 +21,6 @@ import ( "time" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/mediaapi/storage/sqlite3/storage.go b/mediaapi/storage/sqlite3/storage.go index ea81f9120..010c0a66e 100644 --- a/mediaapi/storage/sqlite3/storage.go +++ b/mediaapi/storage/sqlite3/storage.go @@ -20,7 +20,6 @@ import ( "database/sql" // Import the postgres database driver. - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -41,7 +40,7 @@ func Open(dataSourceName string) (*Database, error) { if err != nil { return nil, err } - if d.db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil { + if d.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { return nil, err } if err = d.statements.prepare(d.db); err != nil { diff --git a/mediaapi/storage/sqlite3/thumbnail_table.go b/mediaapi/storage/sqlite3/thumbnail_table.go index a0956f68c..432a1590c 100644 --- a/mediaapi/storage/sqlite3/thumbnail_table.go +++ b/mediaapi/storage/sqlite3/thumbnail_table.go @@ -21,7 +21,6 @@ import ( "time" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/mediaapi/storage/storage.go b/mediaapi/storage/storage.go index 2694e197d..5ff114db6 100644 --- a/mediaapi/storage/storage.go +++ b/mediaapi/storage/storage.go @@ -19,13 +19,13 @@ package storage import ( "net/url" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/storage/postgres" "github.com/matrix-org/dendrite/mediaapi/storage/sqlite3" ) // Open opens a postgres database. -func Open(dataSourceName string, dbProperties internal.DbProperties) (Database, error) { +func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return postgres.Open(dataSourceName, dbProperties) diff --git a/mediaapi/storage/storage_wasm.go b/mediaapi/storage/storage_wasm.go index 78de2cb82..a672271f9 100644 --- a/mediaapi/storage/storage_wasm.go +++ b/mediaapi/storage/storage_wasm.go @@ -18,14 +18,14 @@ import ( "fmt" "net/url" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/storage/sqlite3" ) // Open opens a postgres database. func Open( dataSourceName string, - dbProperties internal.DbProperties, // nolint:unparam + dbProperties sqlutil.DbProperties, // nolint:unparam ) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { diff --git a/publicroomsapi/consumers/roomserver.go b/publicroomsapi/consumers/roomserver.go index c513d3b24..b9686d56d 100644 --- a/publicroomsapi/consumers/roomserver.go +++ b/publicroomsapi/consumers/roomserver.go @@ -78,29 +78,17 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { return nil } - ev := output.NewRoomEvent.Event - log.WithFields(log.Fields{ - "event_id": ev.EventID(), - "room_id": ev.RoomID(), - "type": ev.Type(), - }).Info("received event from roomserver") - - addQueryReq := api.QueryEventsByIDRequest{EventIDs: output.NewRoomEvent.AddsStateEventIDs} - var addQueryRes api.QueryEventsByIDResponse - if err := s.rsAPI.QueryEventsByID(context.TODO(), &addQueryReq, &addQueryRes); err != nil { - log.Warn(err) - return err - } - - remQueryReq := api.QueryEventsByIDRequest{EventIDs: output.NewRoomEvent.RemovesStateEventIDs} var remQueryRes api.QueryEventsByIDResponse - if err := s.rsAPI.QueryEventsByID(context.TODO(), &remQueryReq, &remQueryRes); err != nil { - log.Warn(err) - return err + if len(output.NewRoomEvent.RemovesStateEventIDs) > 0 { + remQueryReq := api.QueryEventsByIDRequest{EventIDs: output.NewRoomEvent.RemovesStateEventIDs} + if err := s.rsAPI.QueryEventsByID(context.TODO(), &remQueryReq, &remQueryRes); err != nil { + log.Warn(err) + return err + } } var addQueryEvents, remQueryEvents []gomatrixserverlib.Event - for _, headeredEvent := range addQueryRes.Events { + for _, headeredEvent := range output.NewRoomEvent.AddsState() { addQueryEvents = append(addQueryEvents, headeredEvent.Event) } for _, headeredEvent := range remQueryRes.Events { diff --git a/publicroomsapi/routing/routing.go b/publicroomsapi/routing/routing.go index ee1434392..2da555f98 100644 --- a/publicroomsapi/routing/routing.go +++ b/publicroomsapi/routing/routing.go @@ -17,13 +17,13 @@ package routing import ( "net/http" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/gorilla/mux" "github.com/matrix-org/dendrite/clientapi/auth" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/publicroomsapi/directory" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/publicroomsapi/types" @@ -51,8 +51,8 @@ func Setup( } r0mux.Handle("/directory/list/room/{roomID}", - internal.MakeExternalAPI("directory_list", func(req *http.Request) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeExternalAPI("directory_list", func(req *http.Request) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -61,8 +61,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) // TODO: Add AS support r0mux.Handle("/directory/list/room/{roomID}", - internal.MakeAuthAPI("directory_list", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("directory_list", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -70,7 +70,7 @@ func Setup( }), ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/publicRooms", - internal.MakeExternalAPI("public_rooms", func(req *http.Request) util.JSONResponse { + httputil.MakeExternalAPI("public_rooms", func(req *http.Request) util.JSONResponse { if extRoomsProvider != nil { return directory.GetPostPublicRoomsWithExternal(req, publicRoomsDB, fedClient, extRoomsProvider) } @@ -80,7 +80,7 @@ func Setup( // Federation - TODO: should this live here or in federation API? It's sure easier if it's here so here it is. publicAPIMux.Handle("/federation/v1/publicRooms", - internal.MakeExternalAPI("federation_public_rooms", func(req *http.Request) util.JSONResponse { + httputil.MakeExternalAPI("federation_public_rooms", func(req *http.Request) util.JSONResponse { return directory.GetPostPublicRooms(req, publicRoomsDB) }), ).Methods(http.MethodGet) diff --git a/publicroomsapi/storage/postgres/storage.go b/publicroomsapi/storage/postgres/storage.go index 5f0f551a7..36c6aec64 100644 --- a/publicroomsapi/storage/postgres/storage.go +++ b/publicroomsapi/storage/postgres/storage.go @@ -20,7 +20,7 @@ import ( "database/sql" "encoding/json" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" @@ -29,7 +29,7 @@ import ( // PublicRoomsServerDatabase represents a public rooms server database. type PublicRoomsServerDatabase struct { db *sql.DB - internal.PartitionOffsetStatements + sqlutil.PartitionOffsetStatements statements publicRoomsStatements localServerName gomatrixserverlib.ServerName } @@ -37,7 +37,7 @@ type PublicRoomsServerDatabase struct { type attributeValue interface{} // NewPublicRoomsServerDatabase creates a new public rooms server database. -func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties internal.DbProperties, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) { +func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) { var db *sql.DB var err error if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { @@ -138,33 +138,33 @@ func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent( case "m.room.aliases": return d.updateRoomAliases(ctx, event) case "m.room.canonical_alias": - var content internal.CanonicalAliasContent + var content eventutil.CanonicalAliasContent field := &(content.Alias) attrName := "canonical_alias" return d.updateStringAttribute(ctx, attrName, event, &content, field) case "m.room.name": - var content internal.NameContent + var content eventutil.NameContent field := &(content.Name) attrName := "name" return d.updateStringAttribute(ctx, attrName, event, &content, field) case "m.room.topic": - var content internal.TopicContent + var content eventutil.TopicContent field := &(content.Topic) attrName := "topic" return d.updateStringAttribute(ctx, attrName, event, &content, field) case "m.room.avatar": - var content internal.AvatarContent + var content eventutil.AvatarContent field := &(content.URL) attrName := "avatar_url" return d.updateStringAttribute(ctx, attrName, event, &content, field) case "m.room.history_visibility": - var content internal.HistoryVisibilityContent + var content eventutil.HistoryVisibilityContent field := &(content.HistoryVisibility) attrName := "world_readable" strForTrue := "world_readable" return d.updateBooleanAttribute(ctx, attrName, event, &content, field, strForTrue) case "m.room.guest_access": - var content internal.GuestAccessContent + var content eventutil.GuestAccessContent field := &(content.GuestAccess) attrName := "guest_can_join" strForTrue := "can_join" @@ -248,7 +248,7 @@ func (d *PublicRoomsServerDatabase) updateRoomAliases( if aliasesEvent.StateKey() == nil || *aliasesEvent.StateKey() != string(d.localServerName) { return nil // only store our own aliases } - var content internal.AliasesContent + var content eventutil.AliasesContent if err := json.Unmarshal(aliasesEvent.Content(), &content); err != nil { return err } diff --git a/publicroomsapi/storage/sqlite3/storage.go b/publicroomsapi/storage/sqlite3/storage.go index f51051823..5c685d131 100644 --- a/publicroomsapi/storage/sqlite3/storage.go +++ b/publicroomsapi/storage/sqlite3/storage.go @@ -22,7 +22,7 @@ import ( _ "github.com/mattn/go-sqlite3" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" @@ -31,7 +31,7 @@ import ( // PublicRoomsServerDatabase represents a public rooms server database. type PublicRoomsServerDatabase struct { db *sql.DB - internal.PartitionOffsetStatements + sqlutil.PartitionOffsetStatements statements publicRoomsStatements localServerName gomatrixserverlib.ServerName } @@ -46,7 +46,7 @@ func NewPublicRoomsServerDatabase(dataSourceName string, localServerName gomatri if err != nil { return nil, err } - if db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil { + if db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { return nil, err } storage := PublicRoomsServerDatabase{ @@ -144,33 +144,33 @@ func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent( case "m.room.aliases": return d.updateRoomAliases(ctx, event) case "m.room.canonical_alias": - var content internal.CanonicalAliasContent + var content eventutil.CanonicalAliasContent field := &(content.Alias) attrName := "canonical_alias" return d.updateStringAttribute(ctx, attrName, event, &content, field) case "m.room.name": - var content internal.NameContent + var content eventutil.NameContent field := &(content.Name) attrName := "name" return d.updateStringAttribute(ctx, attrName, event, &content, field) case "m.room.topic": - var content internal.TopicContent + var content eventutil.TopicContent field := &(content.Topic) attrName := "topic" return d.updateStringAttribute(ctx, attrName, event, &content, field) case "m.room.avatar": - var content internal.AvatarContent + var content eventutil.AvatarContent field := &(content.URL) attrName := "avatar_url" return d.updateStringAttribute(ctx, attrName, event, &content, field) case "m.room.history_visibility": - var content internal.HistoryVisibilityContent + var content eventutil.HistoryVisibilityContent field := &(content.HistoryVisibility) attrName := "world_readable" strForTrue := "world_readable" return d.updateBooleanAttribute(ctx, attrName, event, &content, field, strForTrue) case "m.room.guest_access": - var content internal.GuestAccessContent + var content eventutil.GuestAccessContent field := &(content.GuestAccess) attrName := "guest_can_join" strForTrue := "can_join" @@ -254,7 +254,7 @@ func (d *PublicRoomsServerDatabase) updateRoomAliases( if aliasesEvent.StateKey() == nil || *aliasesEvent.StateKey() != string(d.localServerName) { return nil // only store our own aliases } - var content internal.AliasesContent + var content eventutil.AliasesContent if err := json.Unmarshal(aliasesEvent.Content(), &content); err != nil { return err } diff --git a/publicroomsapi/storage/storage.go b/publicroomsapi/storage/storage.go index 507b9fa6c..f66188040 100644 --- a/publicroomsapi/storage/storage.go +++ b/publicroomsapi/storage/storage.go @@ -19,7 +19,7 @@ package storage import ( "net/url" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/publicroomsapi/storage/postgres" "github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3" "github.com/matrix-org/gomatrixserverlib" @@ -29,7 +29,7 @@ const schemePostgres = "postgres" const schemeFile = "file" // NewPublicRoomsServerDatabase opens a database connection. -func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties internal.DbProperties, localServerName gomatrixserverlib.ServerName) (Database, error) { +func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, localServerName gomatrixserverlib.ServerName) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties, localServerName) diff --git a/roomserver/api/api.go b/roomserver/api/api.go index aefe55bcd..967f58baf 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -65,13 +65,6 @@ type RoomserverInternalAPI interface { response *QueryMembershipsForRoomResponse, ) error - // Query a list of invite event senders for a user in a room. - QueryInvitesForUser( - ctx context.Context, - request *QueryInvitesForUserRequest, - response *QueryInvitesForUserResponse, - ) error - // Query whether a server is allowed to see an event QueryServerAllowedToSeeEvent( ctx context.Context, @@ -96,10 +89,10 @@ type RoomserverInternalAPI interface { ) error // Query a given amount (or less) of events prior to a given set of events. - QueryBackfill( + PerformBackfill( ctx context.Context, - request *QueryBackfillRequest, - response *QueryBackfillResponse, + request *PerformBackfillRequest, + response *PerformBackfillResponse, ) error // Asks for the default room version as preferred by the server. diff --git a/roomserver/api/api_trace.go b/roomserver/api/api_trace.go new file mode 100644 index 000000000..a478eeb9a --- /dev/null +++ b/roomserver/api/api_trace.go @@ -0,0 +1,218 @@ +package api + +import ( + "context" + "encoding/json" + "fmt" + + fsAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/util" +) + +// RoomserverInternalAPITrace wraps a RoomserverInternalAPI and logs the +// complete request/response/error +type RoomserverInternalAPITrace struct { + Impl RoomserverInternalAPI +} + +func (t *RoomserverInternalAPITrace) SetFederationSenderAPI(fsAPI fsAPI.FederationSenderInternalAPI) { + t.Impl.SetFederationSenderAPI(fsAPI) +} + +func (t *RoomserverInternalAPITrace) InputRoomEvents( + ctx context.Context, + req *InputRoomEventsRequest, + res *InputRoomEventsResponse, +) error { + err := t.Impl.InputRoomEvents(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("InputRoomEvents req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) PerformJoin( + ctx context.Context, + req *PerformJoinRequest, + res *PerformJoinResponse, +) error { + err := t.Impl.PerformJoin(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("PerformJoin req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) PerformLeave( + ctx context.Context, + req *PerformLeaveRequest, + res *PerformLeaveResponse, +) error { + err := t.Impl.PerformLeave(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("PerformLeave req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) QueryLatestEventsAndState( + ctx context.Context, + req *QueryLatestEventsAndStateRequest, + res *QueryLatestEventsAndStateResponse, +) error { + err := t.Impl.QueryLatestEventsAndState(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("QueryLatestEventsAndState req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) QueryStateAfterEvents( + ctx context.Context, + req *QueryStateAfterEventsRequest, + res *QueryStateAfterEventsResponse, +) error { + err := t.Impl.QueryStateAfterEvents(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("QueryStateAfterEvents req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) QueryEventsByID( + ctx context.Context, + req *QueryEventsByIDRequest, + res *QueryEventsByIDResponse, +) error { + err := t.Impl.QueryEventsByID(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("QueryEventsByID req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) QueryMembershipForUser( + ctx context.Context, + req *QueryMembershipForUserRequest, + res *QueryMembershipForUserResponse, +) error { + err := t.Impl.QueryMembershipForUser(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("QueryMembershipForUser req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) QueryMembershipsForRoom( + ctx context.Context, + req *QueryMembershipsForRoomRequest, + res *QueryMembershipsForRoomResponse, +) error { + err := t.Impl.QueryMembershipsForRoom(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("QueryMembershipsForRoom req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) QueryServerAllowedToSeeEvent( + ctx context.Context, + req *QueryServerAllowedToSeeEventRequest, + res *QueryServerAllowedToSeeEventResponse, +) error { + err := t.Impl.QueryServerAllowedToSeeEvent(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("QueryServerAllowedToSeeEvent req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) QueryMissingEvents( + ctx context.Context, + req *QueryMissingEventsRequest, + res *QueryMissingEventsResponse, +) error { + err := t.Impl.QueryMissingEvents(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("QueryMissingEvents req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) QueryStateAndAuthChain( + ctx context.Context, + req *QueryStateAndAuthChainRequest, + res *QueryStateAndAuthChainResponse, +) error { + err := t.Impl.QueryStateAndAuthChain(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("QueryStateAndAuthChain req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) PerformBackfill( + ctx context.Context, + req *PerformBackfillRequest, + res *PerformBackfillResponse, +) error { + err := t.Impl.PerformBackfill(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("PerformBackfill req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) QueryRoomVersionCapabilities( + ctx context.Context, + req *QueryRoomVersionCapabilitiesRequest, + res *QueryRoomVersionCapabilitiesResponse, +) error { + err := t.Impl.QueryRoomVersionCapabilities(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("QueryRoomVersionCapabilities req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) QueryRoomVersionForRoom( + ctx context.Context, + req *QueryRoomVersionForRoomRequest, + res *QueryRoomVersionForRoomResponse, +) error { + err := t.Impl.QueryRoomVersionForRoom(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("QueryRoomVersionForRoom req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) SetRoomAlias( + ctx context.Context, + req *SetRoomAliasRequest, + res *SetRoomAliasResponse, +) error { + err := t.Impl.SetRoomAlias(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("SetRoomAlias req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) GetRoomIDForAlias( + ctx context.Context, + req *GetRoomIDForAliasRequest, + res *GetRoomIDForAliasResponse, +) error { + err := t.Impl.GetRoomIDForAlias(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("GetRoomIDForAlias req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) GetAliasesForRoomID( + ctx context.Context, + req *GetAliasesForRoomIDRequest, + res *GetAliasesForRoomIDResponse, +) error { + err := t.Impl.GetAliasesForRoomID(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("GetAliasesForRoomID req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) GetCreatorIDForAlias( + ctx context.Context, + req *GetCreatorIDForAliasRequest, + res *GetCreatorIDForAliasResponse, +) error { + err := t.Impl.GetCreatorIDForAlias(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("GetCreatorIDForAlias req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) RemoveRoomAlias( + ctx context.Context, + req *RemoveRoomAliasRequest, + res *RemoveRoomAliasResponse, +) error { + err := t.Impl.RemoveRoomAlias(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("RemoveRoomAlias req=%+v res=%+v", js(req), js(res)) + return err +} + +func js(thing interface{}) string { + b, err := json.Marshal(thing) + if err != nil { + return fmt.Sprintf("Marshal error:%s", err) + } + return string(b) +} diff --git a/roomserver/api/output.go b/roomserver/api/output.go index 92a468a96..2bbd97af8 100644 --- a/roomserver/api/output.go +++ b/roomserver/api/output.go @@ -63,6 +63,13 @@ type OutputNewRoomEvent struct { // Together with RemovesStateEventIDs this allows the receiver to keep an up to date // view of the current state of the room. AddsStateEventIDs []string `json:"adds_state_event_ids"` + // All extra newly added state events. This is only set if there are *extra* events + // other than `Event`. This can happen when forks get merged because state resolution + // may decide a bunch of state events on one branch are now valid, so they will be + // present in this list. This is useful when trying to maintain the current state of a room + // as to do so you need to include both these events and `Event`. + AddStateEvents []gomatrixserverlib.HeaderedEvent `json:"adds_state_events"` + // The state event IDs that were removed from the state of the room by this event. RemovesStateEventIDs []string `json:"removes_state_event_ids"` // The ID of the event that was output before this event. @@ -112,6 +119,26 @@ type OutputNewRoomEvent struct { TransactionID *TransactionID `json:"transaction_id"` } +// AddsState returns all added state events from this event. +// +// This function is needed because `AddStateEvents` will not include a copy of +// the original event to save space, so you cannot use that slice alone. +// Instead, use this function which will add the original event if it is present +// in `AddsStateEventIDs`. +func (ore *OutputNewRoomEvent) AddsState() []gomatrixserverlib.HeaderedEvent { + includeOutputEvent := false + for _, id := range ore.AddsStateEventIDs { + if id == ore.Event.EventID() { + includeOutputEvent = true + break + } + } + if !includeOutputEvent { + return ore.AddStateEvents + } + return append(ore.AddStateEvents, ore.Event) +} + // An OutputNewInviteEvent is written whenever an invite becomes active. // Invite events can be received outside of an existing room so have to be // tracked separately from the room events themselves. diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go index 1cf54144e..3e5cae1b6 100644 --- a/roomserver/api/perform.go +++ b/roomserver/api/perform.go @@ -2,6 +2,7 @@ package api import ( "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" ) type PerformJoinRequest struct { @@ -22,3 +23,31 @@ type PerformLeaveRequest struct { type PerformLeaveResponse struct { } + +// PerformBackfillRequest is a request to PerformBackfill. +type PerformBackfillRequest struct { + // The room to backfill + RoomID string `json:"room_id"` + // A map of backwards extremity event ID to a list of its prev_event IDs. + BackwardsExtremities map[string][]string `json:"backwards_extremities"` + // The maximum number of events to retrieve. + Limit int `json:"limit"` + // The server interested in the events. + ServerName gomatrixserverlib.ServerName `json:"server_name"` +} + +// PrevEventIDs returns the prev_event IDs of all backwards extremities, de-duplicated in a lexicographically sorted order. +func (r *PerformBackfillRequest) PrevEventIDs() []string { + var prevEventIDs []string + for _, pes := range r.BackwardsExtremities { + prevEventIDs = append(prevEventIDs, pes...) + } + prevEventIDs = util.UniqueStrings(prevEventIDs) + return prevEventIDs +} + +// PerformBackfillResponse is a response to PerformBackfill. +type PerformBackfillResponse struct { + // Missing events, arbritrary order. + Events []gomatrixserverlib.HeaderedEvent `json:"events"` +} diff --git a/roomserver/api/query.go b/roomserver/api/query.go index dc005c77c..6586b1af3 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -18,7 +18,6 @@ package api import ( "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" ) // QueryLatestEventsAndStateRequest is a request to QueryLatestEventsAndState @@ -34,8 +33,6 @@ type QueryLatestEventsAndStateRequest struct { // This is used when sending events to set the prev_events, auth_events and depth. // It is also used to tell whether the event is allowed by the event auth rules. type QueryLatestEventsAndStateResponse struct { - // Copy of the request for debugging. - QueryLatestEventsAndStateRequest // Does the room exist? // If the room doesn't exist this will be false and LatestEvents will be empty. RoomExists bool `json:"room_exists"` @@ -67,8 +64,6 @@ type QueryStateAfterEventsRequest struct { // QueryStateAfterEventsResponse is a response to QueryStateAfterEvents type QueryStateAfterEventsResponse struct { - // Copy of the request for debugging. - QueryStateAfterEventsRequest // Does the room exist on this roomserver? // If the room doesn't exist this will be false and StateEvents will be empty. RoomExists bool `json:"room_exists"` @@ -90,8 +85,6 @@ type QueryEventsByIDRequest struct { // QueryEventsByIDResponse is a response to QueryEventsByID type QueryEventsByIDResponse struct { - // Copy of the request for debugging. - QueryEventsByIDRequest // A list of events with the requested IDs. // If the roomserver does not have a copy of a requested event // then it will omit that event from the list. @@ -140,23 +133,6 @@ type QueryMembershipsForRoomResponse struct { HasBeenInRoom bool `json:"has_been_in_room"` } -// QueryInvitesForUserRequest is a request to QueryInvitesForUser -type QueryInvitesForUserRequest struct { - // The room ID to look up invites in. - RoomID string `json:"room_id"` - // The User ID to look up invites for. - TargetUserID string `json:"target_user_id"` -} - -// QueryInvitesForUserResponse is a response to QueryInvitesForUser -// This is used when accepting an invite or rejecting a invite to tell which -// remote matrix servers to contact. -type QueryInvitesForUserResponse struct { - // A list of matrix user IDs for each sender of an active invite targeting - // the requested user ID. - InviteSenderUserIDs []string `json:"invite_sender_user_ids"` -} - // QueryServerAllowedToSeeEventRequest is a request to QueryServerAllowedToSeeEvent type QueryServerAllowedToSeeEventRequest struct { // The event ID to look up invites in. @@ -205,8 +181,6 @@ type QueryStateAndAuthChainRequest struct { // QueryStateAndAuthChainResponse is a response to QueryStateAndAuthChain type QueryStateAndAuthChainResponse struct { - // Copy of the request for debugging. - QueryStateAndAuthChainRequest // Does the room exist on this roomserver? // If the room doesn't exist this will be false and StateEvents will be empty. RoomExists bool `json:"room_exists"` @@ -221,34 +195,6 @@ type QueryStateAndAuthChainResponse struct { AuthChainEvents []gomatrixserverlib.HeaderedEvent `json:"auth_chain_events"` } -// QueryBackfillRequest is a request to QueryBackfill. -type QueryBackfillRequest struct { - // The room to backfill - RoomID string `json:"room_id"` - // A map of backwards extremity event ID to a list of its prev_event IDs. - BackwardsExtremities map[string][]string `json:"backwards_extremities"` - // The maximum number of events to retrieve. - Limit int `json:"limit"` - // The server interested in the events. - ServerName gomatrixserverlib.ServerName `json:"server_name"` -} - -// PrevEventIDs returns the prev_event IDs of all backwards extremities, de-duplicated in a lexicographically sorted order. -func (r *QueryBackfillRequest) PrevEventIDs() []string { - var prevEventIDs []string - for _, pes := range r.BackwardsExtremities { - prevEventIDs = append(prevEventIDs, pes...) - } - prevEventIDs = util.UniqueStrings(prevEventIDs) - return prevEventIDs -} - -// QueryBackfillResponse is a response to QueryBackfill. -type QueryBackfillResponse struct { - // Missing events, arbritrary order. - Events []gomatrixserverlib.HeaderedEvent `json:"events"` -} - // QueryRoomVersionCapabilitiesRequest asks for the default room version type QueryRoomVersionCapabilitiesRequest struct{} diff --git a/roomserver/internal/input.go b/roomserver/internal/input.go index 932b4df46..e863af953 100644 --- a/roomserver/internal/input.go +++ b/roomserver/internal/input.go @@ -21,6 +21,7 @@ import ( "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/roomserver/api" + log "github.com/sirupsen/logrus" fsAPI "github.com/matrix-org/dendrite/federationsender/api" ) @@ -40,6 +41,21 @@ func (r *RoomserverInternalAPI) WriteOutputEvents(roomID string, updates []api.O if err != nil { return err } + logger := log.WithFields(log.Fields{ + "room_id": roomID, + "type": updates[i].Type, + }) + if updates[i].NewRoomEvent != nil { + logger = logger.WithFields(log.Fields{ + "event_type": updates[i].NewRoomEvent.Event.Type(), + "event_id": updates[i].NewRoomEvent.Event.EventID(), + "adds_state": len(updates[i].NewRoomEvent.AddsStateEventIDs), + "removes_state": len(updates[i].NewRoomEvent.RemovesStateEventIDs), + "send_as_server": updates[i].NewRoomEvent.SendAsServer, + "sender": updates[i].NewRoomEvent.Event.Sender(), + }) + } + logger.Infof("Producing to topic '%s'", r.OutputRoomEventTopic) messages[i] = &sarama.ProducerMessage{ Topic: r.OutputRoomEventTopic, Key: sarama.StringEncoder(roomID), diff --git a/roomserver/internal/input_events.go b/roomserver/internal/input_events.go index 3a38636f8..4487aea02 100644 --- a/roomserver/internal/input_events.go +++ b/roomserver/internal/input_events.go @@ -21,7 +21,7 @@ import ( "errors" "fmt" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/storage" @@ -177,7 +177,7 @@ func (r *RoomserverInternalAPI) processInviteEvent( } succeeded := false defer func() { - txerr := internal.EndTransaction(updater, &succeeded) + txerr := sqlutil.EndTransaction(updater, &succeeded) if err == nil && txerr != nil { err = txerr } diff --git a/roomserver/internal/input_latest_events.go b/roomserver/internal/input_latest_events.go index aea85ca95..66316ac4f 100644 --- a/roomserver/internal/input_latest_events.go +++ b/roomserver/internal/input_latest_events.go @@ -19,8 +19,9 @@ package internal import ( "bytes" "context" + "fmt" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/types" @@ -59,7 +60,7 @@ func (r *RoomserverInternalAPI) updateLatestEvents( } succeeded := false defer func() { - txerr := internal.EndTransaction(updater, &succeeded) + txerr := sqlutil.EndTransaction(updater, &succeeded) if err == nil && txerr != nil { err = txerr } @@ -310,24 +311,11 @@ func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error) TransactionID: u.transactionID, } - var stateEventNIDs []types.EventNID - for _, entry := range u.added { - stateEventNIDs = append(stateEventNIDs, entry.EventNID) - } - for _, entry := range u.removed { - stateEventNIDs = append(stateEventNIDs, entry.EventNID) - } - for _, entry := range u.stateBeforeEventRemoves { - stateEventNIDs = append(stateEventNIDs, entry.EventNID) - } - for _, entry := range u.stateBeforeEventAdds { - stateEventNIDs = append(stateEventNIDs, entry.EventNID) - } - stateEventNIDs = stateEventNIDs[:util.SortAndUnique(eventNIDSorter(stateEventNIDs))] - eventIDMap, err := u.api.DB.EventIDs(u.ctx, stateEventNIDs) + eventIDMap, err := u.stateEventMap() if err != nil { return nil, err } + for _, entry := range u.added { ore.AddsStateEventIDs = append(ore.AddsStateEventIDs, eventIDMap[entry.EventNID]) } @@ -342,12 +330,60 @@ func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error) } ore.SendAsServer = u.sendAsServer + // include extra state events if they were added as nearly every downstream component will care about it + // and we'd rather not have them all hit QueryEventsByID at the same time! + if len(ore.AddsStateEventIDs) > 0 { + ore.AddStateEvents, err = u.extraEventsForIDs(roomVersion, ore.AddsStateEventIDs) + if err != nil { + return nil, fmt.Errorf("failed to load add_state_events from db: %w", err) + } + } + return &api.OutputEvent{ Type: api.OutputTypeNewRoomEvent, NewRoomEvent: &ore, }, nil } +// extraEventsForIDs returns the full events for the event IDs given, but does not include the current event being +// updated. +func (u *latestEventsUpdater) extraEventsForIDs(roomVersion gomatrixserverlib.RoomVersion, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error) { + var extraEventIDs []string + for _, e := range eventIDs { + if e == u.event.EventID() { + continue + } + extraEventIDs = append(extraEventIDs, e) + } + if len(extraEventIDs) == 0 { + return nil, nil + } + extraEvents, err := u.api.DB.EventsFromIDs(u.ctx, extraEventIDs) + if err != nil { + return nil, err + } + var h []gomatrixserverlib.HeaderedEvent + for _, e := range extraEvents { + h = append(h, e.Headered(roomVersion)) + } + return h, nil +} + +// retrieve an event nid -> event ID map for all events that need updating +func (u *latestEventsUpdater) stateEventMap() (map[types.EventNID]string, error) { + var stateEventNIDs []types.EventNID + var allStateEntries []types.StateEntry + allStateEntries = append(allStateEntries, u.added...) + allStateEntries = append(allStateEntries, u.removed...) + allStateEntries = append(allStateEntries, u.stateBeforeEventRemoves...) + allStateEntries = append(allStateEntries, u.stateBeforeEventAdds...) + for _, entry := range allStateEntries { + stateEventNIDs = append(stateEventNIDs, entry.EventNID) + } + stateEventNIDs = stateEventNIDs[:util.SortAndUnique(eventNIDSorter(stateEventNIDs))] + return u.api.DB.EventIDs(u.ctx, stateEventNIDs) +} + type eventNIDSorter []types.EventNID func (s eventNIDSorter) Len() int { return len(s) } diff --git a/roomserver/internal/query_backfill.go b/roomserver/internal/perform_backfill.go similarity index 100% rename from roomserver/internal/query_backfill.go rename to roomserver/internal/perform_backfill.go diff --git a/roomserver/internal/perform_join.go b/roomserver/internal/perform_join.go index 75dfdd19c..1c951bb15 100644 --- a/roomserver/internal/perform_join.go +++ b/roomserver/internal/perform_join.go @@ -7,7 +7,7 @@ import ( "time" fsAPI "github.com/matrix-org/dendrite/federationsender/api" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" @@ -159,7 +159,7 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( // TODO: Check what happens if the room exists on the server // but everyone has since left. I suspect it does the wrong thing. buildRes := api.QueryLatestEventsAndStateResponse{} - event, err := internal.BuildEvent( + event, err := eventutil.BuildEvent( ctx, // the request context &eb, // the template join event r.Cfg, // the server configuration @@ -202,7 +202,7 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( } } - case internal.ErrRoomNoExists: + case eventutil.ErrRoomNoExists: // The room doesn't exist. First of all check if the room is a local // room. If it is then there's nothing more to do - the room just // hasn't been created yet. diff --git a/roomserver/internal/perform_leave.go b/roomserver/internal/perform_leave.go index b60c2e99f..880c8b203 100644 --- a/roomserver/internal/perform_leave.go +++ b/roomserver/internal/perform_leave.go @@ -7,7 +7,7 @@ import ( "time" fsAPI "github.com/matrix-org/dendrite/federationsender/api" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" ) @@ -97,7 +97,7 @@ func (r *RoomserverInternalAPI) performLeaveRoomByID( // TODO: Check what happens if the room exists on the server // but everyone has since left. I suspect it does the wrong thing. buildRes := api.QueryLatestEventsAndStateResponse{} - event, err := internal.BuildEvent( + event, err := eventutil.BuildEvent( ctx, // the request context &eb, // the template leave event r.Cfg, // the server configuration @@ -106,7 +106,7 @@ func (r *RoomserverInternalAPI) performLeaveRoomByID( &buildRes, // the query response ) if err != nil { - return fmt.Errorf("internal.BuildEvent: %w", err) + return fmt.Errorf("eventutil.BuildEvent: %w", err) } // Give our leave event to the roomserver input stream. The diff --git a/roomserver/internal/query.go b/roomserver/internal/query.go index 9fb67e7eb..4fc8e4c25 100644 --- a/roomserver/internal/query.go +++ b/roomserver/internal/query.go @@ -45,7 +45,6 @@ func (r *RoomserverInternalAPI) QueryLatestEventsAndState( roomState := state.NewStateResolution(r.DB) - response.QueryLatestEventsAndStateRequest = *request roomNID, err := r.DB.RoomNIDExcludingStubs(ctx, request.RoomID) if err != nil { return err @@ -105,7 +104,6 @@ func (r *RoomserverInternalAPI) QueryStateAfterEvents( roomState := state.NewStateResolution(r.DB) - response.QueryStateAfterEventsRequest = *request roomNID, err := r.DB.RoomNIDExcludingStubs(ctx, request.RoomID) if err != nil { return err @@ -153,8 +151,6 @@ func (r *RoomserverInternalAPI) QueryEventsByID( request *api.QueryEventsByIDRequest, response *api.QueryEventsByIDResponse, ) error { - response.QueryEventsByIDRequest = *request - eventNIDMap, err := r.DB.EventNIDs(ctx, request.EventIDs) if err != nil { return err @@ -353,40 +349,6 @@ func getMembershipsAtState( return events, nil } -// QueryInvitesForUser implements api.RoomserverInternalAPI -func (r *RoomserverInternalAPI) QueryInvitesForUser( - ctx context.Context, - request *api.QueryInvitesForUserRequest, - response *api.QueryInvitesForUserResponse, -) error { - roomNID, err := r.DB.RoomNID(ctx, request.RoomID) - if err != nil { - return err - } - - targetUserNIDs, err := r.DB.EventStateKeyNIDs(ctx, []string{request.TargetUserID}) - if err != nil { - return err - } - targetUserNID := targetUserNIDs[request.TargetUserID] - - senderUserNIDs, err := r.DB.GetInvitesForUser(ctx, roomNID, targetUserNID) - if err != nil { - return err - } - - senderUserIDs, err := r.DB.EventStateKeys(ctx, senderUserNIDs) - if err != nil { - return err - } - - for _, senderUserID := range senderUserIDs { - response.InviteSenderUserIDs = append(response.InviteSenderUserIDs, senderUserID) - } - - return nil -} - // QueryServerAllowedToSeeEvent implements api.RoomserverInternalAPI func (r *RoomserverInternalAPI) QueryServerAllowedToSeeEvent( ctx context.Context, @@ -475,11 +437,11 @@ func (r *RoomserverInternalAPI) QueryMissingEvents( return err } -// QueryBackfill implements api.RoomServerQueryAPI -func (r *RoomserverInternalAPI) QueryBackfill( +// PerformBackfill implements api.RoomServerQueryAPI +func (r *RoomserverInternalAPI) PerformBackfill( ctx context.Context, - request *api.QueryBackfillRequest, - response *api.QueryBackfillResponse, + request *api.PerformBackfillRequest, + response *api.PerformBackfillResponse, ) error { // if we are requesting the backfill then we need to do a federation hit // TODO: we could be more sensible and fetch as many events we already have then request the rest @@ -523,7 +485,7 @@ func (r *RoomserverInternalAPI) QueryBackfill( return err } -func (r *RoomserverInternalAPI) backfillViaFederation(ctx context.Context, req *api.QueryBackfillRequest, res *api.QueryBackfillResponse) error { +func (r *RoomserverInternalAPI) backfillViaFederation(ctx context.Context, req *api.PerformBackfillRequest, res *api.PerformBackfillResponse) error { roomVer, err := r.DB.GetRoomVersionForRoom(ctx, req.RoomID) if err != nil { return fmt.Errorf("backfillViaFederation: unknown room version for room %s : %w", req.RoomID, err) @@ -681,7 +643,7 @@ func (r *RoomserverInternalAPI) scanEventTree( var pre string // TODO: add tests for this function to ensure it meets the contract that callers expect (and doc what that is supposed to be) - // Currently, callers like QueryBackfill will call scanEventTree with a pre-populated `visited` map, assuming that by doing + // Currently, callers like PerformBackfill will call scanEventTree with a pre-populated `visited` map, assuming that by doing // so means that the events in that map will NOT be returned from this function. That is not currently true, resulting in // duplicate events being sent in response to /backfill requests. initialIgnoreList := make(map[string]bool, len(visited)) @@ -768,7 +730,6 @@ func (r *RoomserverInternalAPI) QueryStateAndAuthChain( request *api.QueryStateAndAuthChainRequest, response *api.QueryStateAndAuthChainResponse, ) error { - response.QueryStateAndAuthChainRequest = *request roomNID, err := r.DB.RoomNIDExcludingStubs(ctx, request.RoomID) if err != nil { return err diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go index 5cc2537e3..e41adb993 100644 --- a/roomserver/inthttp/client.go +++ b/roomserver/inthttp/client.go @@ -7,7 +7,7 @@ import ( fsInputAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal/caching" - internalHTTP "github.com/matrix-org/dendrite/internal/http" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/opentracing/opentracing-go" ) @@ -24,8 +24,9 @@ const ( RoomserverInputRoomEventsPath = "/roomserver/inputRoomEvents" // Perform operations - RoomserverPerformJoinPath = "/roomserver/performJoin" - RoomserverPerformLeavePath = "/roomserver/performLeave" + RoomserverPerformJoinPath = "/roomserver/performJoin" + RoomserverPerformLeavePath = "/roomserver/performLeave" + RoomserverPerformBackfillPath = "/roomserver/performBackfill" // Query operations RoomserverQueryLatestEventsAndStatePath = "/roomserver/queryLatestEventsAndState" @@ -33,11 +34,9 @@ const ( RoomserverQueryEventsByIDPath = "/roomserver/queryEventsByID" RoomserverQueryMembershipForUserPath = "/roomserver/queryMembershipForUser" RoomserverQueryMembershipsForRoomPath = "/roomserver/queryMembershipsForRoom" - RoomserverQueryInvitesForUserPath = "/roomserver/queryInvitesForUser" RoomserverQueryServerAllowedToSeeEventPath = "/roomserver/queryServerAllowedToSeeEvent" RoomserverQueryMissingEventsPath = "/roomserver/queryMissingEvents" RoomserverQueryStateAndAuthChainPath = "/roomserver/queryStateAndAuthChain" - RoomserverQueryBackfillPath = "/roomserver/queryBackfill" RoomserverQueryRoomVersionCapabilitiesPath = "/roomserver/queryRoomVersionCapabilities" RoomserverQueryRoomVersionForRoomPath = "/roomserver/queryRoomVersionForRoom" ) @@ -79,7 +78,7 @@ func (h *httpRoomserverInternalAPI) SetRoomAlias( defer span.Finish() apiURL := h.roomserverURL + RoomserverSetRoomAliasPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // GetRoomIDForAlias implements RoomserverAliasAPI @@ -92,7 +91,7 @@ func (h *httpRoomserverInternalAPI) GetRoomIDForAlias( defer span.Finish() apiURL := h.roomserverURL + RoomserverGetRoomIDForAliasPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // GetAliasesForRoomID implements RoomserverAliasAPI @@ -105,7 +104,7 @@ func (h *httpRoomserverInternalAPI) GetAliasesForRoomID( defer span.Finish() apiURL := h.roomserverURL + RoomserverGetAliasesForRoomIDPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // GetCreatorIDForAlias implements RoomserverAliasAPI @@ -118,7 +117,7 @@ func (h *httpRoomserverInternalAPI) GetCreatorIDForAlias( defer span.Finish() apiURL := h.roomserverURL + RoomserverGetCreatorIDForAliasPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // RemoveRoomAlias implements RoomserverAliasAPI @@ -131,7 +130,7 @@ func (h *httpRoomserverInternalAPI) RemoveRoomAlias( defer span.Finish() apiURL := h.roomserverURL + RoomserverRemoveRoomAliasPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // InputRoomEvents implements RoomserverInputAPI @@ -144,7 +143,7 @@ func (h *httpRoomserverInternalAPI) InputRoomEvents( defer span.Finish() apiURL := h.roomserverURL + RoomserverInputRoomEventsPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } func (h *httpRoomserverInternalAPI) PerformJoin( @@ -156,7 +155,7 @@ func (h *httpRoomserverInternalAPI) PerformJoin( defer span.Finish() apiURL := h.roomserverURL + RoomserverPerformJoinPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } func (h *httpRoomserverInternalAPI) PerformLeave( @@ -168,7 +167,7 @@ func (h *httpRoomserverInternalAPI) PerformLeave( defer span.Finish() apiURL := h.roomserverURL + RoomserverPerformLeavePath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryLatestEventsAndState implements RoomserverQueryAPI @@ -181,7 +180,7 @@ func (h *httpRoomserverInternalAPI) QueryLatestEventsAndState( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryLatestEventsAndStatePath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryStateAfterEvents implements RoomserverQueryAPI @@ -194,7 +193,7 @@ func (h *httpRoomserverInternalAPI) QueryStateAfterEvents( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryStateAfterEventsPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryEventsByID implements RoomserverQueryAPI @@ -207,7 +206,7 @@ func (h *httpRoomserverInternalAPI) QueryEventsByID( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryEventsByIDPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryMembershipForUser implements RoomserverQueryAPI @@ -220,7 +219,7 @@ func (h *httpRoomserverInternalAPI) QueryMembershipForUser( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryMembershipForUserPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryMembershipsForRoom implements RoomserverQueryAPI @@ -233,20 +232,7 @@ func (h *httpRoomserverInternalAPI) QueryMembershipsForRoom( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryMembershipsForRoomPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// QueryInvitesForUser implements RoomserverQueryAPI -func (h *httpRoomserverInternalAPI) QueryInvitesForUser( - ctx context.Context, - request *api.QueryInvitesForUserRequest, - response *api.QueryInvitesForUserResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryInvitesForUser") - defer span.Finish() - - apiURL := h.roomserverURL + RoomserverQueryInvitesForUserPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryServerAllowedToSeeEvent implements RoomserverQueryAPI @@ -259,7 +245,7 @@ func (h *httpRoomserverInternalAPI) QueryServerAllowedToSeeEvent( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryServerAllowedToSeeEventPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryMissingEvents implements RoomServerQueryAPI @@ -272,7 +258,7 @@ func (h *httpRoomserverInternalAPI) QueryMissingEvents( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryMissingEventsPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryStateAndAuthChain implements RoomserverQueryAPI @@ -285,20 +271,20 @@ func (h *httpRoomserverInternalAPI) QueryStateAndAuthChain( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryStateAndAuthChainPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } -// QueryBackfill implements RoomServerQueryAPI -func (h *httpRoomserverInternalAPI) QueryBackfill( +// PerformBackfill implements RoomServerQueryAPI +func (h *httpRoomserverInternalAPI) PerformBackfill( ctx context.Context, - request *api.QueryBackfillRequest, - response *api.QueryBackfillResponse, + request *api.PerformBackfillRequest, + response *api.PerformBackfillResponse, ) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryBackfill") + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformBackfill") defer span.Finish() - apiURL := h.roomserverURL + RoomserverQueryBackfillPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + apiURL := h.roomserverURL + RoomserverPerformBackfillPath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryRoomVersionCapabilities implements RoomServerQueryAPI @@ -311,7 +297,7 @@ func (h *httpRoomserverInternalAPI) QueryRoomVersionCapabilities( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryRoomVersionCapabilitiesPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryRoomVersionForRoom implements RoomServerQueryAPI @@ -329,7 +315,7 @@ func (h *httpRoomserverInternalAPI) QueryRoomVersionForRoom( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryRoomVersionForRoomPath - err := internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) if err == nil { h.cache.StoreRoomVersion(request.RoomID, response.RoomVersion) } diff --git a/roomserver/inthttp/server.go b/roomserver/inthttp/server.go index 9a58a30b0..822acd15b 100644 --- a/roomserver/inthttp/server.go +++ b/roomserver/inthttp/server.go @@ -5,7 +5,7 @@ import ( "net/http" "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/util" ) @@ -14,7 +14,7 @@ import ( // nolint: gocyclo func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { internalAPIMux.Handle(RoomserverInputRoomEventsPath, - internal.MakeInternalAPI("inputRoomEvents", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("inputRoomEvents", func(req *http.Request) util.JSONResponse { var request api.InputRoomEventsRequest var response api.InputRoomEventsResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -27,7 +27,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { }), ) internalAPIMux.Handle(RoomserverPerformJoinPath, - internal.MakeInternalAPI("performJoin", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("performJoin", func(req *http.Request) util.JSONResponse { var request api.PerformJoinRequest var response api.PerformJoinResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -40,7 +40,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { }), ) internalAPIMux.Handle(RoomserverPerformLeavePath, - internal.MakeInternalAPI("performLeave", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("performLeave", func(req *http.Request) util.JSONResponse { var request api.PerformLeaveRequest var response api.PerformLeaveResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -54,7 +54,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverQueryLatestEventsAndStatePath, - internal.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse { var request api.QueryLatestEventsAndStateRequest var response api.QueryLatestEventsAndStateResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -68,7 +68,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverQueryStateAfterEventsPath, - internal.MakeInternalAPI("queryStateAfterEvents", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("queryStateAfterEvents", func(req *http.Request) util.JSONResponse { var request api.QueryStateAfterEventsRequest var response api.QueryStateAfterEventsResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -82,7 +82,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverQueryEventsByIDPath, - internal.MakeInternalAPI("queryEventsByID", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("queryEventsByID", func(req *http.Request) util.JSONResponse { var request api.QueryEventsByIDRequest var response api.QueryEventsByIDResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -96,7 +96,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverQueryMembershipForUserPath, - internal.MakeInternalAPI("QueryMembershipForUser", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("QueryMembershipForUser", func(req *http.Request) util.JSONResponse { var request api.QueryMembershipForUserRequest var response api.QueryMembershipForUserResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -110,7 +110,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverQueryMembershipsForRoomPath, - internal.MakeInternalAPI("queryMembershipsForRoom", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("queryMembershipsForRoom", func(req *http.Request) util.JSONResponse { var request api.QueryMembershipsForRoomRequest var response api.QueryMembershipsForRoomResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -122,23 +122,9 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - internalAPIMux.Handle( - RoomserverQueryInvitesForUserPath, - internal.MakeInternalAPI("queryInvitesForUser", func(req *http.Request) util.JSONResponse { - var request api.QueryInvitesForUserRequest - var response api.QueryInvitesForUserResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryInvitesForUser(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) internalAPIMux.Handle( RoomserverQueryServerAllowedToSeeEventPath, - internal.MakeInternalAPI("queryServerAllowedToSeeEvent", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("queryServerAllowedToSeeEvent", func(req *http.Request) util.JSONResponse { var request api.QueryServerAllowedToSeeEventRequest var response api.QueryServerAllowedToSeeEventResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -152,7 +138,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverQueryMissingEventsPath, - internal.MakeInternalAPI("queryMissingEvents", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("queryMissingEvents", func(req *http.Request) util.JSONResponse { var request api.QueryMissingEventsRequest var response api.QueryMissingEventsResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -166,7 +152,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverQueryStateAndAuthChainPath, - internal.MakeInternalAPI("queryStateAndAuthChain", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("queryStateAndAuthChain", func(req *http.Request) util.JSONResponse { var request api.QueryStateAndAuthChainRequest var response api.QueryStateAndAuthChainResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -179,14 +165,14 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { }), ) internalAPIMux.Handle( - RoomserverQueryBackfillPath, - internal.MakeInternalAPI("QueryBackfill", func(req *http.Request) util.JSONResponse { - var request api.QueryBackfillRequest - var response api.QueryBackfillResponse + RoomserverPerformBackfillPath, + httputil.MakeInternalAPI("PerformBackfill", func(req *http.Request) util.JSONResponse { + var request api.PerformBackfillRequest + var response api.PerformBackfillResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { return util.ErrorResponse(err) } - if err := r.QueryBackfill(req.Context(), &request, &response); err != nil { + if err := r.PerformBackfill(req.Context(), &request, &response); err != nil { return util.ErrorResponse(err) } return util.JSONResponse{Code: http.StatusOK, JSON: &response} @@ -194,7 +180,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverQueryRoomVersionCapabilitiesPath, - internal.MakeInternalAPI("QueryRoomVersionCapabilities", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("QueryRoomVersionCapabilities", func(req *http.Request) util.JSONResponse { var request api.QueryRoomVersionCapabilitiesRequest var response api.QueryRoomVersionCapabilitiesResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -208,7 +194,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverQueryRoomVersionForRoomPath, - internal.MakeInternalAPI("QueryRoomVersionForRoom", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("QueryRoomVersionForRoom", func(req *http.Request) util.JSONResponse { var request api.QueryRoomVersionForRoomRequest var response api.QueryRoomVersionForRoomResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -222,7 +208,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverSetRoomAliasPath, - internal.MakeInternalAPI("setRoomAlias", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("setRoomAlias", func(req *http.Request) util.JSONResponse { var request api.SetRoomAliasRequest var response api.SetRoomAliasResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -236,7 +222,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverGetRoomIDForAliasPath, - internal.MakeInternalAPI("GetRoomIDForAlias", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("GetRoomIDForAlias", func(req *http.Request) util.JSONResponse { var request api.GetRoomIDForAliasRequest var response api.GetRoomIDForAliasResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -250,7 +236,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverGetCreatorIDForAliasPath, - internal.MakeInternalAPI("GetCreatorIDForAlias", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("GetCreatorIDForAlias", func(req *http.Request) util.JSONResponse { var request api.GetCreatorIDForAliasRequest var response api.GetCreatorIDForAliasResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -264,7 +250,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverGetAliasesForRoomIDPath, - internal.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse { var request api.GetAliasesForRoomIDRequest var response api.GetAliasesForRoomIDResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -278,7 +264,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverRemoveRoomAliasPath, - internal.MakeInternalAPI("removeRoomAlias", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("removeRoomAlias", func(req *http.Request) util.JSONResponse { var request api.RemoveRoomAliasRequest var response api.RemoveRoomAliasResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index a9db22d7a..427d5ff36 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -20,7 +20,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/inthttp" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/roomserver/internal" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/sirupsen/logrus" @@ -35,7 +35,7 @@ func AddInternalRoutes(router *mux.Router, intAPI api.RoomserverInternalAPI) { // NewInternalAPI returns a concerete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( - base *basecomponent.BaseDendrite, + base *setup.BaseDendrite, keyRing gomatrixserverlib.JSONVerifier, fedClient *gomatrixserverlib.FederationClient, ) api.RoomserverInternalAPI { diff --git a/roomserver/storage/postgres/event_json_table.go b/roomserver/storage/postgres/event_json_table.go index 3bce9f086..7df175954 100644 --- a/roomserver/storage/postgres/event_json_table.go +++ b/roomserver/storage/postgres/event_json_table.go @@ -20,7 +20,6 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" diff --git a/roomserver/storage/postgres/event_state_keys_table.go b/roomserver/storage/postgres/event_state_keys_table.go index b7e2cb6b7..500ff20e4 100644 --- a/roomserver/storage/postgres/event_state_keys_table.go +++ b/roomserver/storage/postgres/event_state_keys_table.go @@ -21,6 +21,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -94,7 +95,7 @@ func (s *eventStateKeyStatements) InsertEventStateKeyNID( ctx context.Context, txn *sql.Tx, eventStateKey string, ) (types.EventStateKeyNID, error) { var eventStateKeyNID int64 - stmt := internal.TxStmt(txn, s.insertEventStateKeyNIDStmt) + stmt := sqlutil.TxStmt(txn, s.insertEventStateKeyNIDStmt) err := stmt.QueryRowContext(ctx, eventStateKey).Scan(&eventStateKeyNID) return types.EventStateKeyNID(eventStateKeyNID), err } @@ -103,7 +104,7 @@ func (s *eventStateKeyStatements) SelectEventStateKeyNID( ctx context.Context, txn *sql.Tx, eventStateKey string, ) (types.EventStateKeyNID, error) { var eventStateKeyNID int64 - stmt := internal.TxStmt(txn, s.selectEventStateKeyNIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectEventStateKeyNIDStmt) err := stmt.QueryRowContext(ctx, eventStateKey).Scan(&eventStateKeyNID) return types.EventStateKeyNID(eventStateKeyNID), err } diff --git a/roomserver/storage/postgres/event_types_table.go b/roomserver/storage/postgres/event_types_table.go index f8a7bff1b..037d98fe7 100644 --- a/roomserver/storage/postgres/event_types_table.go +++ b/roomserver/storage/postgres/event_types_table.go @@ -19,9 +19,8 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/internal" - "github.com/lib/pq" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go index d39e9511a..bdbf5e7cb 100644 --- a/roomserver/storage/postgres/events_table.go +++ b/roomserver/storage/postgres/events_table.go @@ -22,6 +22,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -284,13 +285,13 @@ func (s *eventStatements) UpdateEventState( func (s *eventStatements) SelectEventSentToOutput( ctx context.Context, txn *sql.Tx, eventNID types.EventNID, ) (sentToOutput bool, err error) { - stmt := internal.TxStmt(txn, s.selectEventSentToOutputStmt) + stmt := sqlutil.TxStmt(txn, s.selectEventSentToOutputStmt) err = stmt.QueryRowContext(ctx, int64(eventNID)).Scan(&sentToOutput) return } func (s *eventStatements) UpdateEventSentToOutput(ctx context.Context, txn *sql.Tx, eventNID types.EventNID) error { - stmt := internal.TxStmt(txn, s.updateEventSentToOutputStmt) + stmt := sqlutil.TxStmt(txn, s.updateEventSentToOutputStmt) _, err := stmt.ExecContext(ctx, int64(eventNID)) return err } @@ -298,7 +299,7 @@ func (s *eventStatements) UpdateEventSentToOutput(ctx context.Context, txn *sql. func (s *eventStatements) SelectEventID( ctx context.Context, txn *sql.Tx, eventNID types.EventNID, ) (eventID string, err error) { - stmt := internal.TxStmt(txn, s.selectEventIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectEventIDStmt) err = stmt.QueryRowContext(ctx, int64(eventNID)).Scan(&eventID) return } @@ -306,7 +307,7 @@ func (s *eventStatements) SelectEventID( func (s *eventStatements) BulkSelectStateAtEventAndReference( ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID, ) ([]types.StateAtEventAndReference, error) { - stmt := internal.TxStmt(txn, s.bulkSelectStateAtEventAndReferenceStmt) + stmt := sqlutil.TxStmt(txn, s.bulkSelectStateAtEventAndReferenceStmt) rows, err := stmt.QueryContext(ctx, eventNIDsAsArray(eventNIDs)) if err != nil { return nil, err diff --git a/roomserver/storage/postgres/invite_table.go b/roomserver/storage/postgres/invite_table.go index 55a5bc7d7..048a094dc 100644 --- a/roomserver/storage/postgres/invite_table.go +++ b/roomserver/storage/postgres/invite_table.go @@ -20,6 +20,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -101,7 +102,7 @@ func (s *inviteStatements) InsertInviteEvent( targetUserNID, senderUserNID types.EventStateKeyNID, inviteEventJSON []byte, ) (bool, error) { - result, err := internal.TxStmt(txn, s.insertInviteEventStmt).ExecContext( + result, err := sqlutil.TxStmt(txn, s.insertInviteEventStmt).ExecContext( ctx, inviteEventID, roomNID, targetUserNID, senderUserNID, inviteEventJSON, ) if err != nil { @@ -118,7 +119,7 @@ func (s *inviteStatements) UpdateInviteRetired( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, ) ([]string, error) { - stmt := internal.TxStmt(txn, s.updateInviteRetiredStmt) + stmt := sqlutil.TxStmt(txn, s.updateInviteRetiredStmt) rows, err := stmt.QueryContext(ctx, roomNID, targetUserNID) if err != nil { return nil, err diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go index c635ce28a..13cef638f 100644 --- a/roomserver/storage/postgres/membership_table.go +++ b/roomserver/storage/postgres/membership_table.go @@ -20,6 +20,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -133,7 +134,7 @@ func (s *membershipStatements) InsertMembership( txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, localTarget bool, ) error { - stmt := internal.TxStmt(txn, s.insertMembershipStmt) + stmt := sqlutil.TxStmt(txn, s.insertMembershipStmt) _, err := stmt.ExecContext(ctx, roomNID, targetUserNID, localTarget) return err } @@ -142,7 +143,7 @@ func (s *membershipStatements) SelectMembershipForUpdate( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, ) (membership tables.MembershipState, err error) { - err = internal.TxStmt(txn, s.selectMembershipForUpdateStmt).QueryRowContext( + err = sqlutil.TxStmt(txn, s.selectMembershipForUpdateStmt).QueryRowContext( ctx, roomNID, targetUserNID, ).Scan(&membership) return @@ -216,7 +217,7 @@ func (s *membershipStatements) UpdateMembership( senderUserNID types.EventStateKeyNID, membership tables.MembershipState, eventNID types.EventNID, ) error { - _, err := internal.TxStmt(txn, s.updateMembershipStmt).ExecContext( + _, err := sqlutil.TxStmt(txn, s.updateMembershipStmt).ExecContext( ctx, roomNID, targetUserNID, senderUserNID, membership, eventNID, ) return err diff --git a/roomserver/storage/postgres/previous_events_table.go b/roomserver/storage/postgres/previous_events_table.go index ff0306967..1a4ba6732 100644 --- a/roomserver/storage/postgres/previous_events_table.go +++ b/roomserver/storage/postgres/previous_events_table.go @@ -19,7 +19,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -85,7 +85,7 @@ func (s *previousEventStatements) InsertPreviousEvent( previousEventReferenceSHA256 []byte, eventNID types.EventNID, ) error { - stmt := internal.TxStmt(txn, s.insertPreviousEventStmt) + stmt := sqlutil.TxStmt(txn, s.insertPreviousEventStmt) _, err := stmt.ExecContext( ctx, previousEventID, previousEventReferenceSHA256, int64(eventNID), ) @@ -98,6 +98,6 @@ func (s *previousEventStatements) SelectPreviousEventExists( ctx context.Context, txn *sql.Tx, eventID string, eventReferenceSHA256 []byte, ) error { var ok int64 - stmt := internal.TxStmt(txn, s.selectPreviousEventExistsStmt) + stmt := sqlutil.TxStmt(txn, s.selectPreviousEventExistsStmt) return stmt.QueryRowContext(ctx, eventID, eventReferenceSHA256).Scan(&ok) } diff --git a/roomserver/storage/postgres/rooms_table.go b/roomserver/storage/postgres/rooms_table.go index 5a353ed1c..8e00cfdb8 100644 --- a/roomserver/storage/postgres/rooms_table.go +++ b/roomserver/storage/postgres/rooms_table.go @@ -21,7 +21,7 @@ import ( "errors" "github.com/lib/pq" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -106,7 +106,7 @@ func (s *roomStatements) InsertRoomNID( roomID string, roomVersion gomatrixserverlib.RoomVersion, ) (types.RoomNID, error) { var roomNID int64 - stmt := internal.TxStmt(txn, s.insertRoomNIDStmt) + stmt := sqlutil.TxStmt(txn, s.insertRoomNIDStmt) err := stmt.QueryRowContext(ctx, roomID, roomVersion).Scan(&roomNID) return types.RoomNID(roomNID), err } @@ -115,7 +115,7 @@ func (s *roomStatements) SelectRoomNID( ctx context.Context, txn *sql.Tx, roomID string, ) (types.RoomNID, error) { var roomNID int64 - stmt := internal.TxStmt(txn, s.selectRoomNIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectRoomNIDStmt) err := stmt.QueryRowContext(ctx, roomID).Scan(&roomNID) return types.RoomNID(roomNID), err } @@ -143,7 +143,7 @@ func (s *roomStatements) SelectLatestEventsNIDsForUpdate( var nids pq.Int64Array var lastEventSentNID int64 var stateSnapshotNID int64 - stmt := internal.TxStmt(txn, s.selectLatestEventNIDsForUpdateStmt) + stmt := sqlutil.TxStmt(txn, s.selectLatestEventNIDsForUpdateStmt) err := stmt.QueryRowContext(ctx, int64(roomNID)).Scan(&nids, &lastEventSentNID, &stateSnapshotNID) if err != nil { return nil, 0, 0, err @@ -163,7 +163,7 @@ func (s *roomStatements) UpdateLatestEventNIDs( lastEventSentNID types.EventNID, stateSnapshotNID types.StateSnapshotNID, ) error { - stmt := internal.TxStmt(txn, s.updateLatestEventNIDsStmt) + stmt := sqlutil.TxStmt(txn, s.updateLatestEventNIDsStmt) _, err := stmt.ExecContext( ctx, roomNID, @@ -178,7 +178,7 @@ func (s *roomStatements) SelectRoomVersionForRoomID( ctx context.Context, txn *sql.Tx, roomID string, ) (gomatrixserverlib.RoomVersion, error) { var roomVersion gomatrixserverlib.RoomVersion - stmt := internal.TxStmt(txn, s.selectRoomVersionForRoomIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectRoomVersionForRoomIDStmt) err := stmt.QueryRowContext(ctx, roomID).Scan(&roomVersion) if err == sql.ErrNoRows { return roomVersion, errors.New("room not found") diff --git a/roomserver/storage/postgres/state_block_table.go b/roomserver/storage/postgres/state_block_table.go index dbe21d989..d618686f7 100644 --- a/roomserver/storage/postgres/state_block_table.go +++ b/roomserver/storage/postgres/state_block_table.go @@ -21,9 +21,8 @@ import ( "fmt" "sort" - "github.com/matrix-org/dendrite/internal" - "github.com/lib/pq" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index 971f2b9e6..d76ee0a92 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -18,7 +18,6 @@ package postgres import ( "database/sql" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" // Import the postgres database driver. @@ -33,7 +32,7 @@ type Database struct { // Open a postgres database. // nolint: gocyclo -func Open(dataSourceName string, dbProperties internal.DbProperties) (*Database, error) { +func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database, error) { var d Database var db *sql.DB var err error diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index bb5b51562..2751cc557 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -5,7 +5,7 @@ import ( "database/sql" "encoding/json" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -68,7 +68,7 @@ func (d *Database) AddState( stateBlockNIDs []types.StateBlockNID, state []types.StateEntry, ) (stateNID types.StateSnapshotNID, err error) { - err = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.DB, func(txn *sql.Tx) error { if len(state) > 0 { var stateBlockNID types.StateBlockNID stateBlockNID, err = d.StateBlockTable.BulkInsertStateData(ctx, txn, state) @@ -158,7 +158,7 @@ func (d *Database) RoomNIDExcludingStubs(ctx context.Context, roomID string) (ro func (d *Database) LatestEventIDs( ctx context.Context, roomNID types.RoomNID, ) (references []gomatrixserverlib.EventReference, currentStateSnapshotNID types.StateSnapshotNID, depth int64, err error) { - err = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.DB, func(txn *sql.Tx) error { var eventNIDs []types.EventNID eventNIDs, currentStateSnapshotNID, err = d.RoomsTable.SelectLatestEventNIDs(ctx, txn, roomNID) if err != nil { @@ -337,7 +337,7 @@ func (d *Database) StoreEvent( err error ) - err = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.DB, func(txn *sql.Tx) error { if txnAndSessionID != nil { if err = d.TransactionsTable.InsertTransaction( ctx, txn, txnAndSessionID.TransactionID, diff --git a/roomserver/storage/sqlite3/event_json_table.go b/roomserver/storage/sqlite3/event_json_table.go index 7ff4c1d4d..da0c448dc 100644 --- a/roomserver/storage/sqlite3/event_json_table.go +++ b/roomserver/storage/sqlite3/event_json_table.go @@ -21,6 +21,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -69,7 +70,7 @@ func NewSqliteEventJSONTable(db *sql.DB) (tables.EventJSON, error) { func (s *eventJSONStatements) InsertEventJSON( ctx context.Context, txn *sql.Tx, eventNID types.EventNID, eventJSON []byte, ) error { - _, err := internal.TxStmt(txn, s.insertEventJSONStmt).ExecContext(ctx, int64(eventNID), eventJSON) + _, err := sqlutil.TxStmt(txn, s.insertEventJSONStmt).ExecContext(ctx, int64(eventNID), eventJSON) return err } @@ -80,7 +81,7 @@ func (s *eventJSONStatements) BulkSelectEventJSON( for k, v := range eventNIDs { iEventNIDs[k] = v } - selectOrig := strings.Replace(bulkSelectEventJSONSQL, "($1)", internal.QueryVariadic(len(iEventNIDs)), 1) + selectOrig := strings.Replace(bulkSelectEventJSONSQL, "($1)", sqlutil.QueryVariadic(len(iEventNIDs)), 1) rows, err := s.db.QueryContext(ctx, selectOrig, iEventNIDs...) if err != nil { diff --git a/roomserver/storage/sqlite3/event_state_keys_table.go b/roomserver/storage/sqlite3/event_state_keys_table.go index 2d2c04d26..cbea8428c 100644 --- a/roomserver/storage/sqlite3/event_state_keys_table.go +++ b/roomserver/storage/sqlite3/event_state_keys_table.go @@ -21,6 +21,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -90,7 +91,7 @@ func (s *eventStateKeyStatements) InsertEventStateKeyNID( var eventStateKeyNID int64 var err error var res sql.Result - insertStmt := internal.TxStmt(txn, s.insertEventStateKeyNIDStmt) + insertStmt := sqlutil.TxStmt(txn, s.insertEventStateKeyNIDStmt) if res, err = insertStmt.ExecContext(ctx, eventStateKey); err == nil { eventStateKeyNID, err = res.LastInsertId() } @@ -101,7 +102,7 @@ func (s *eventStateKeyStatements) SelectEventStateKeyNID( ctx context.Context, txn *sql.Tx, eventStateKey string, ) (types.EventStateKeyNID, error) { var eventStateKeyNID int64 - stmt := internal.TxStmt(txn, s.selectEventStateKeyNIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectEventStateKeyNIDStmt) err := stmt.QueryRowContext(ctx, eventStateKey).Scan(&eventStateKeyNID) return types.EventStateKeyNID(eventStateKeyNID), err } @@ -113,7 +114,7 @@ func (s *eventStateKeyStatements) BulkSelectEventStateKeyNID( for k, v := range eventStateKeys { iEventStateKeys[k] = v } - selectOrig := strings.Replace(bulkSelectEventStateKeySQL, "($1)", internal.QueryVariadic(len(eventStateKeys)), 1) + selectOrig := strings.Replace(bulkSelectEventStateKeySQL, "($1)", sqlutil.QueryVariadic(len(eventStateKeys)), 1) rows, err := s.db.QueryContext(ctx, selectOrig, iEventStateKeys...) if err != nil { @@ -139,7 +140,7 @@ func (s *eventStateKeyStatements) BulkSelectEventStateKey( for k, v := range eventStateKeyNIDs { iEventStateKeyNIDs[k] = v } - selectOrig := strings.Replace(bulkSelectEventStateKeyNIDSQL, "($1)", internal.QueryVariadic(len(eventStateKeyNIDs)), 1) + selectOrig := strings.Replace(bulkSelectEventStateKeyNIDSQL, "($1)", sqlutil.QueryVariadic(len(eventStateKeyNIDs)), 1) rows, err := s.db.QueryContext(ctx, selectOrig, iEventStateKeyNIDs...) if err != nil { diff --git a/roomserver/storage/sqlite3/event_types_table.go b/roomserver/storage/sqlite3/event_types_table.go index 6d229bbd7..c9a461f99 100644 --- a/roomserver/storage/sqlite3/event_types_table.go +++ b/roomserver/storage/sqlite3/event_types_table.go @@ -21,6 +21,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -104,8 +105,8 @@ func (s *eventTypeStatements) InsertEventTypeNID( ) (types.EventTypeNID, error) { var eventTypeNID int64 var err error - insertStmt := internal.TxStmt(tx, s.insertEventTypeNIDStmt) - resultStmt := internal.TxStmt(tx, s.insertEventTypeNIDResultStmt) + insertStmt := sqlutil.TxStmt(tx, s.insertEventTypeNIDStmt) + resultStmt := sqlutil.TxStmt(tx, s.insertEventTypeNIDResultStmt) if _, err = insertStmt.ExecContext(ctx, eventType); err == nil { err = resultStmt.QueryRowContext(ctx).Scan(&eventTypeNID) } @@ -116,7 +117,7 @@ func (s *eventTypeStatements) SelectEventTypeNID( ctx context.Context, tx *sql.Tx, eventType string, ) (types.EventTypeNID, error) { var eventTypeNID int64 - selectStmt := internal.TxStmt(tx, s.selectEventTypeNIDStmt) + selectStmt := sqlutil.TxStmt(tx, s.selectEventTypeNIDStmt) err := selectStmt.QueryRowContext(ctx, eventType).Scan(&eventTypeNID) return types.EventTypeNID(eventTypeNID), err } @@ -129,7 +130,7 @@ func (s *eventTypeStatements) BulkSelectEventTypeNID( for k, v := range eventTypes { iEventTypes[k] = v } - selectOrig := strings.Replace(bulkSelectEventTypeNIDSQL, "($1)", internal.QueryVariadic(len(iEventTypes)), 1) + selectOrig := strings.Replace(bulkSelectEventTypeNIDSQL, "($1)", sqlutil.QueryVariadic(len(iEventTypes)), 1) selectPrep, err := s.db.Prepare(selectOrig) if err != nil { return nil, err diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index 491399ce7..d66db4694 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -23,6 +23,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -150,7 +151,7 @@ func (s *eventStatements) InsertEvent( depth int64, ) (types.EventNID, types.StateSnapshotNID, error) { // attempt to insert: the last_row_id is the event NID - insertStmt := internal.TxStmt(txn, s.insertEventStmt) + insertStmt := sqlutil.TxStmt(txn, s.insertEventStmt) result, err := insertStmt.ExecContext( ctx, int64(roomNID), int64(eventTypeNID), int64(eventStateKeyNID), eventID, referenceSHA256, eventNIDsAsArray(authEventNIDs), depth, @@ -171,7 +172,7 @@ func (s *eventStatements) SelectEvent( ) (types.EventNID, types.StateSnapshotNID, error) { var eventNID int64 var stateNID int64 - selectStmt := internal.TxStmt(txn, s.selectEventStmt) + selectStmt := sqlutil.TxStmt(txn, s.selectEventStmt) err := selectStmt.QueryRowContext(ctx, eventID).Scan(&eventNID, &stateNID) return types.EventNID(eventNID), types.StateSnapshotNID(stateNID), err } @@ -186,7 +187,7 @@ func (s *eventStatements) BulkSelectStateEventByID( for k, v := range eventIDs { iEventIDs[k] = v } - selectOrig := strings.Replace(bulkSelectStateEventByIDSQL, "($1)", internal.QueryVariadic(len(iEventIDs)), 1) + selectOrig := strings.Replace(bulkSelectStateEventByIDSQL, "($1)", sqlutil.QueryVariadic(len(iEventIDs)), 1) selectStmt, err := s.db.Prepare(selectOrig) if err != nil { return nil, err @@ -238,7 +239,7 @@ func (s *eventStatements) BulkSelectStateAtEventByID( for k, v := range eventIDs { iEventIDs[k] = v } - selectOrig := strings.Replace(bulkSelectStateAtEventByIDSQL, "($1)", internal.QueryVariadic(len(iEventIDs)), 1) + selectOrig := strings.Replace(bulkSelectStateAtEventByIDSQL, "($1)", sqlutil.QueryVariadic(len(iEventIDs)), 1) selectStmt, err := s.db.Prepare(selectOrig) if err != nil { return nil, err @@ -285,7 +286,7 @@ func (s *eventStatements) UpdateEventState( func (s *eventStatements) SelectEventSentToOutput( ctx context.Context, txn *sql.Tx, eventNID types.EventNID, ) (sentToOutput bool, err error) { - selectStmt := internal.TxStmt(txn, s.selectEventSentToOutputStmt) + selectStmt := sqlutil.TxStmt(txn, s.selectEventSentToOutputStmt) err = selectStmt.QueryRowContext(ctx, int64(eventNID)).Scan(&sentToOutput) //err = s.selectEventSentToOutputStmt.QueryRowContext(ctx, int64(eventNID)).Scan(&sentToOutput) if err != nil { @@ -294,7 +295,7 @@ func (s *eventStatements) SelectEventSentToOutput( } func (s *eventStatements) UpdateEventSentToOutput(ctx context.Context, txn *sql.Tx, eventNID types.EventNID) error { - updateStmt := internal.TxStmt(txn, s.updateEventSentToOutputStmt) + updateStmt := sqlutil.TxStmt(txn, s.updateEventSentToOutputStmt) _, err := updateStmt.ExecContext(ctx, int64(eventNID)) //_, err := s.updateEventSentToOutputStmt.ExecContext(ctx, int64(eventNID)) return err @@ -303,7 +304,7 @@ func (s *eventStatements) UpdateEventSentToOutput(ctx context.Context, txn *sql. func (s *eventStatements) SelectEventID( ctx context.Context, txn *sql.Tx, eventNID types.EventNID, ) (eventID string, err error) { - selectStmt := internal.TxStmt(txn, s.selectEventIDStmt) + selectStmt := sqlutil.TxStmt(txn, s.selectEventIDStmt) err = selectStmt.QueryRowContext(ctx, int64(eventNID)).Scan(&eventID) return } @@ -316,7 +317,7 @@ func (s *eventStatements) BulkSelectStateAtEventAndReference( for k, v := range eventNIDs { iEventNIDs[k] = v } - selectOrig := strings.Replace(bulkSelectStateAtEventAndReferenceSQL, "($1)", internal.QueryVariadic(len(iEventNIDs)), 1) + selectOrig := strings.Replace(bulkSelectStateAtEventAndReferenceSQL, "($1)", sqlutil.QueryVariadic(len(iEventNIDs)), 1) ////////////// rows, err := txn.QueryContext(ctx, selectOrig, iEventNIDs...) @@ -362,14 +363,14 @@ func (s *eventStatements) BulkSelectEventReference( for k, v := range eventNIDs { iEventNIDs[k] = v } - selectOrig := strings.Replace(bulkSelectEventReferenceSQL, "($1)", internal.QueryVariadic(len(iEventNIDs)), 1) + selectOrig := strings.Replace(bulkSelectEventReferenceSQL, "($1)", sqlutil.QueryVariadic(len(iEventNIDs)), 1) selectPrep, err := txn.Prepare(selectOrig) if err != nil { return nil, err } /////////////// - selectStmt := internal.TxStmt(txn, selectPrep) + selectStmt := sqlutil.TxStmt(txn, selectPrep) rows, err := selectStmt.QueryContext(ctx, iEventNIDs...) if err != nil { return nil, err @@ -396,7 +397,7 @@ func (s *eventStatements) BulkSelectEventID(ctx context.Context, eventNIDs []typ for k, v := range eventNIDs { iEventNIDs[k] = v } - selectOrig := strings.Replace(bulkSelectEventIDSQL, "($1)", internal.QueryVariadic(len(iEventNIDs)), 1) + selectOrig := strings.Replace(bulkSelectEventIDSQL, "($1)", sqlutil.QueryVariadic(len(iEventNIDs)), 1) selectStmt, err := s.db.Prepare(selectOrig) if err != nil { return nil, err @@ -432,7 +433,7 @@ func (s *eventStatements) BulkSelectEventNID(ctx context.Context, eventIDs []str for k, v := range eventIDs { iEventIDs[k] = v } - selectOrig := strings.Replace(bulkSelectEventNIDSQL, "($1)", internal.QueryVariadic(len(iEventIDs)), 1) + selectOrig := strings.Replace(bulkSelectEventNIDSQL, "($1)", sqlutil.QueryVariadic(len(iEventIDs)), 1) selectStmt, err := s.db.Prepare(selectOrig) if err != nil { return nil, err @@ -461,7 +462,7 @@ func (s *eventStatements) SelectMaxEventDepth(ctx context.Context, txn *sql.Tx, for i, v := range eventNIDs { iEventIDs[i] = v } - sqlStr := strings.Replace(selectMaxEventDepthSQL, "($1)", internal.QueryVariadic(len(iEventIDs)), 1) + sqlStr := strings.Replace(selectMaxEventDepthSQL, "($1)", sqlutil.QueryVariadic(len(iEventIDs)), 1) err := txn.QueryRowContext(ctx, sqlStr, iEventIDs...).Scan(&result) if err != nil { return 0, err diff --git a/roomserver/storage/sqlite3/invite_table.go b/roomserver/storage/sqlite3/invite_table.go index 7dcc2dc0b..21745d1b0 100644 --- a/roomserver/storage/sqlite3/invite_table.go +++ b/roomserver/storage/sqlite3/invite_table.go @@ -20,6 +20,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -89,7 +90,7 @@ func (s *inviteStatements) InsertInviteEvent( targetUserNID, senderUserNID types.EventStateKeyNID, inviteEventJSON []byte, ) (bool, error) { - stmt := internal.TxStmt(txn, s.insertInviteEventStmt) + stmt := sqlutil.TxStmt(txn, s.insertInviteEventStmt) result, err := stmt.ExecContext( ctx, inviteEventID, roomNID, targetUserNID, senderUserNID, inviteEventJSON, ) @@ -108,7 +109,7 @@ func (s *inviteStatements) UpdateInviteRetired( txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, ) (eventIDs []string, err error) { // gather all the event IDs we will retire - stmt := internal.TxStmt(txn, s.selectInvitesAboutToRetireStmt) + stmt := sqlutil.TxStmt(txn, s.selectInvitesAboutToRetireStmt) rows, err := stmt.QueryContext(ctx, roomNID, targetUserNID) if err != nil { return nil, err @@ -123,7 +124,7 @@ func (s *inviteStatements) UpdateInviteRetired( } // now retire the invites - stmt = internal.TxStmt(txn, s.updateInviteRetiredStmt) + stmt = sqlutil.TxStmt(txn, s.updateInviteRetiredStmt) _, err = stmt.ExecContext(ctx, roomNID, targetUserNID) return } diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go index 5f3076c5a..6f0d763e7 100644 --- a/roomserver/storage/sqlite3/membership_table.go +++ b/roomserver/storage/sqlite3/membership_table.go @@ -20,6 +20,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -109,7 +110,7 @@ func (s *membershipStatements) InsertMembership( roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, localTarget bool, ) error { - stmt := internal.TxStmt(txn, s.insertMembershipStmt) + stmt := sqlutil.TxStmt(txn, s.insertMembershipStmt) _, err := stmt.ExecContext(ctx, roomNID, targetUserNID, localTarget) return err } @@ -118,7 +119,7 @@ func (s *membershipStatements) SelectMembershipForUpdate( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, ) (membership tables.MembershipState, err error) { - stmt := internal.TxStmt(txn, s.selectMembershipForUpdateStmt) + stmt := sqlutil.TxStmt(txn, s.selectMembershipForUpdateStmt) err = stmt.QueryRowContext( ctx, roomNID, targetUserNID, ).Scan(&membership) @@ -193,7 +194,7 @@ func (s *membershipStatements) UpdateMembership( senderUserNID types.EventStateKeyNID, membership tables.MembershipState, eventNID types.EventNID, ) error { - stmt := internal.TxStmt(txn, s.updateMembershipStmt) + stmt := sqlutil.TxStmt(txn, s.updateMembershipStmt) _, err := stmt.ExecContext( ctx, senderUserNID, membership, eventNID, roomNID, targetUserNID, ) diff --git a/roomserver/storage/sqlite3/previous_events_table.go b/roomserver/storage/sqlite3/previous_events_table.go index 2b187bba3..549aecfb7 100644 --- a/roomserver/storage/sqlite3/previous_events_table.go +++ b/roomserver/storage/sqlite3/previous_events_table.go @@ -19,7 +19,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -77,7 +77,7 @@ func (s *previousEventStatements) InsertPreviousEvent( previousEventReferenceSHA256 []byte, eventNID types.EventNID, ) error { - stmt := internal.TxStmt(txn, s.insertPreviousEventStmt) + stmt := sqlutil.TxStmt(txn, s.insertPreviousEventStmt) _, err := stmt.ExecContext( ctx, previousEventID, previousEventReferenceSHA256, int64(eventNID), ) @@ -90,6 +90,6 @@ func (s *previousEventStatements) SelectPreviousEventExists( ctx context.Context, txn *sql.Tx, eventID string, eventReferenceSHA256 []byte, ) error { var ok int64 - stmt := internal.TxStmt(txn, s.selectPreviousEventExistsStmt) + stmt := sqlutil.TxStmt(txn, s.selectPreviousEventExistsStmt) return stmt.QueryRowContext(ctx, eventID, eventReferenceSHA256).Scan(&ok) } diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go index d22fceb3b..ab695c5d2 100644 --- a/roomserver/storage/sqlite3/rooms_table.go +++ b/roomserver/storage/sqlite3/rooms_table.go @@ -21,7 +21,7 @@ import ( "encoding/json" "errors" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -95,7 +95,7 @@ func (s *roomStatements) InsertRoomNID( roomID string, roomVersion gomatrixserverlib.RoomVersion, ) (types.RoomNID, error) { var err error - insertStmt := internal.TxStmt(txn, s.insertRoomNIDStmt) + insertStmt := sqlutil.TxStmt(txn, s.insertRoomNIDStmt) if _, err = insertStmt.ExecContext(ctx, roomID, roomVersion); err == nil { return s.SelectRoomNID(ctx, txn, roomID) } else { @@ -107,7 +107,7 @@ func (s *roomStatements) SelectRoomNID( ctx context.Context, txn *sql.Tx, roomID string, ) (types.RoomNID, error) { var roomNID int64 - stmt := internal.TxStmt(txn, s.selectRoomNIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectRoomNIDStmt) err := stmt.QueryRowContext(ctx, roomID).Scan(&roomNID) return types.RoomNID(roomNID), err } @@ -118,7 +118,7 @@ func (s *roomStatements) SelectLatestEventNIDs( var eventNIDs []types.EventNID var nidsJSON string var stateSnapshotNID int64 - stmt := internal.TxStmt(txn, s.selectLatestEventNIDsStmt) + stmt := sqlutil.TxStmt(txn, s.selectLatestEventNIDsStmt) err := stmt.QueryRowContext(ctx, int64(roomNID)).Scan(&nidsJSON, &stateSnapshotNID) if err != nil { return nil, 0, err @@ -136,7 +136,7 @@ func (s *roomStatements) SelectLatestEventsNIDsForUpdate( var nidsJSON string var lastEventSentNID int64 var stateSnapshotNID int64 - stmt := internal.TxStmt(txn, s.selectLatestEventNIDsForUpdateStmt) + stmt := sqlutil.TxStmt(txn, s.selectLatestEventNIDsForUpdateStmt) err := stmt.QueryRowContext(ctx, int64(roomNID)).Scan(&nidsJSON, &lastEventSentNID, &stateSnapshotNID) if err != nil { return nil, 0, 0, err @@ -155,7 +155,7 @@ func (s *roomStatements) UpdateLatestEventNIDs( lastEventSentNID types.EventNID, stateSnapshotNID types.StateSnapshotNID, ) error { - stmt := internal.TxStmt(txn, s.updateLatestEventNIDsStmt) + stmt := sqlutil.TxStmt(txn, s.updateLatestEventNIDsStmt) _, err := stmt.ExecContext( ctx, eventNIDsAsArray(eventNIDs), @@ -170,7 +170,7 @@ func (s *roomStatements) SelectRoomVersionForRoomID( ctx context.Context, txn *sql.Tx, roomID string, ) (gomatrixserverlib.RoomVersion, error) { var roomVersion gomatrixserverlib.RoomVersion - stmt := internal.TxStmt(txn, s.selectRoomVersionForRoomIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectRoomVersionForRoomIDStmt) err := stmt.QueryRowContext(ctx, roomID).Scan(&roomVersion) if err == sql.ErrNoRows { return roomVersion, errors.New("room not found") diff --git a/roomserver/storage/sqlite3/state_block_table.go b/roomserver/storage/sqlite3/state_block_table.go index 399fdbf63..c058c783a 100644 --- a/roomserver/storage/sqlite3/state_block_table.go +++ b/roomserver/storage/sqlite3/state_block_table.go @@ -23,6 +23,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -130,7 +131,7 @@ func (s *stateBlockStatements) BulkSelectStateBlockEntries( for k, v := range stateBlockNIDs { nids[k] = v } - selectOrig := strings.Replace(bulkSelectStateBlockEntriesSQL, "($1)", internal.QueryVariadic(len(nids)), 1) + selectOrig := strings.Replace(bulkSelectStateBlockEntriesSQL, "($1)", sqlutil.QueryVariadic(len(nids)), 1) selectStmt, err := s.db.Prepare(selectOrig) if err != nil { return nil, err @@ -186,9 +187,9 @@ func (s *stateBlockStatements) BulkSelectFilteredStateBlockEntries( sort.Sort(tuples) eventTypeNIDArray, eventStateKeyNIDArray := tuples.typesAndStateKeysAsArrays() - sqlStatement := strings.Replace(bulkSelectFilteredStateBlockEntriesSQL, "($1)", internal.QueryVariadic(len(stateBlockNIDs)), 1) - sqlStatement = strings.Replace(sqlStatement, "($2)", internal.QueryVariadicOffset(len(eventTypeNIDArray), len(stateBlockNIDs)), 1) - sqlStatement = strings.Replace(sqlStatement, "($3)", internal.QueryVariadicOffset(len(eventStateKeyNIDArray), len(stateBlockNIDs)+len(eventTypeNIDArray)), 1) + sqlStatement := strings.Replace(bulkSelectFilteredStateBlockEntriesSQL, "($1)", sqlutil.QueryVariadic(len(stateBlockNIDs)), 1) + sqlStatement = strings.Replace(sqlStatement, "($2)", sqlutil.QueryVariadicOffset(len(eventTypeNIDArray), len(stateBlockNIDs)), 1) + sqlStatement = strings.Replace(sqlStatement, "($3)", sqlutil.QueryVariadicOffset(len(eventStateKeyNIDArray), len(stateBlockNIDs)+len(eventTypeNIDArray)), 1) var params []interface{} for _, val := range stateBlockNIDs { diff --git a/roomserver/storage/sqlite3/state_snapshot_table.go b/roomserver/storage/sqlite3/state_snapshot_table.go index 5de3f639a..d077b6171 100644 --- a/roomserver/storage/sqlite3/state_snapshot_table.go +++ b/roomserver/storage/sqlite3/state_snapshot_table.go @@ -23,6 +23,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -92,7 +93,7 @@ func (s *stateSnapshotStatements) BulkSelectStateBlockNIDs( for k, v := range stateNIDs { nids[k] = v } - selectOrig := strings.Replace(bulkSelectStateBlockNIDsSQL, "($1)", internal.QueryVariadic(len(nids)), 1) + selectOrig := strings.Replace(bulkSelectStateBlockNIDsSQL, "($1)", sqlutil.QueryVariadic(len(nids)), 1) selectStmt, err := s.db.Prepare(selectOrig) if err != nil { return nil, err diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index 5596b5e3c..8e9352192 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -20,8 +20,6 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal/sqlutil" - - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -52,7 +50,7 @@ func Open(dataSourceName string) (*Database, error) { if err != nil { return nil, err } - if d.db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil { + if d.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { return nil, err } //d.db.Exec("PRAGMA journal_mode=WAL;") diff --git a/roomserver/storage/sqlite3/transactions_table.go b/roomserver/storage/sqlite3/transactions_table.go index 7b489ac6e..1e8de1ca8 100644 --- a/roomserver/storage/sqlite3/transactions_table.go +++ b/roomserver/storage/sqlite3/transactions_table.go @@ -19,7 +19,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" ) @@ -68,7 +68,7 @@ func (s *transactionStatements) InsertTransaction( userID string, eventID string, ) (err error) { - stmt := internal.TxStmt(txn, s.insertTransactionStmt) + stmt := sqlutil.TxStmt(txn, s.insertTransactionStmt) _, err = stmt.ExecContext( ctx, transactionID, sessionID, userID, eventID, ) diff --git a/roomserver/storage/storage.go b/roomserver/storage/storage.go index 071e2cf1d..d7367e4c7 100644 --- a/roomserver/storage/storage.go +++ b/roomserver/storage/storage.go @@ -19,13 +19,13 @@ package storage import ( "net/url" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/postgres" "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" ) // Open opens a database connection. -func Open(dataSourceName string, dbProperties internal.DbProperties) (Database, error) { +func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return postgres.Open(dataSourceName, dbProperties) diff --git a/roomserver/storage/storage_wasm.go b/roomserver/storage/storage_wasm.go index ef30ca593..78405b20e 100644 --- a/roomserver/storage/storage_wasm.go +++ b/roomserver/storage/storage_wasm.go @@ -18,14 +18,14 @@ import ( "fmt" "net/url" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" ) // NewPublicRoomsServerDatabase opens a database connection. func Open( dataSourceName string, - dbProperties internal.DbProperties, // nolint:unparam + dbProperties sqlutil.DbProperties, // nolint:unparam ) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { diff --git a/roomserver/types/types.go b/roomserver/types/types.go index 87a4f935c..241e1e15d 100644 --- a/roomserver/types/types.go +++ b/roomserver/types/types.go @@ -16,7 +16,7 @@ package types import ( - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -174,7 +174,7 @@ type RoomRecentEventsUpdater interface { // It will share the same transaction as this updater. MembershipUpdater(targetUserNID EventStateKeyNID, isTargetLocalUser bool) (MembershipUpdater, error) // Implements Transaction so it can be committed or rolledback - internal.Transaction + sqlutil.Transaction } // A MembershipUpdater is used to update the membership of a user in a room. @@ -199,7 +199,7 @@ type MembershipUpdater interface { // Returns a list of invite event IDs that this state change retired. SetToLeave(senderUserID string, eventID string) (inviteEventIDs []string, err error) // Implements Transaction so it can be committed or rolledback. - internal.Transaction + sqlutil.Transaction } // A MissingEventError is an error that happened because the roomserver was diff --git a/serverkeyapi/inthttp/client.go b/serverkeyapi/inthttp/client.go index d4bc811d6..dfdd936c4 100644 --- a/serverkeyapi/inthttp/client.go +++ b/serverkeyapi/inthttp/client.go @@ -6,7 +6,7 @@ import ( "net/http" "github.com/matrix-org/dendrite/internal/caching" - internalHTTP "github.com/matrix-org/dendrite/internal/http" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/serverkeyapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/opentracing/opentracing-go" @@ -117,7 +117,7 @@ func (h *httpServerKeyInternalAPI) InputPublicKeys( defer span.Finish() apiURL := h.serverKeyAPIURL + ServerKeyInputPublicKeyPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } func (h *httpServerKeyInternalAPI) QueryPublicKeys( @@ -129,7 +129,7 @@ func (h *httpServerKeyInternalAPI) QueryPublicKeys( defer span.Finish() apiURL := h.serverKeyAPIURL + ServerKeyQueryPublicKeyPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } func (h *httpServerKeyInternalAPI) QueryLocalKeys( diff --git a/serverkeyapi/inthttp/server.go b/serverkeyapi/inthttp/server.go index 7bbf191be..c811f7bf8 100644 --- a/serverkeyapi/inthttp/server.go +++ b/serverkeyapi/inthttp/server.go @@ -5,8 +5,8 @@ import ( "net/http" "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/serverkeyapi/api" "github.com/matrix-org/util" ) @@ -26,7 +26,7 @@ func AddRoutes(s api.ServerKeyInternalAPI, internalAPIMux *mux.Router, cache cac }), ) internalAPIMux.Handle(ServerKeyQueryPublicKeyPath, - internal.MakeInternalAPI("queryPublicKeys", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("queryPublicKeys", func(req *http.Request) util.JSONResponse { request := api.QueryPublicKeysRequest{} response := api.QueryPublicKeysResponse{} if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -41,7 +41,7 @@ func AddRoutes(s api.ServerKeyInternalAPI, internalAPIMux *mux.Router, cache cac }), ) internalAPIMux.Handle(ServerKeyInputPublicKeyPath, - internal.MakeInternalAPI("inputPublicKeys", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("inputPublicKeys", func(req *http.Request) util.JSONResponse { request := api.InputPublicKeysRequest{} response := api.InputPublicKeysResponse{} if err := json.NewDecoder(req.Body).Decode(&request); err != nil { diff --git a/serverkeyapi/storage/keydb.go b/serverkeyapi/storage/keydb.go index b9389bd63..c28c4de1e 100644 --- a/serverkeyapi/storage/keydb.go +++ b/serverkeyapi/storage/keydb.go @@ -21,7 +21,7 @@ import ( "golang.org/x/crypto/ed25519" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/serverkeyapi/storage/postgres" "github.com/matrix-org/dendrite/serverkeyapi/storage/sqlite3" "github.com/matrix-org/gomatrixserverlib" @@ -30,7 +30,7 @@ import ( // NewDatabase opens a database connection. func NewDatabase( dataSourceName string, - dbProperties internal.DbProperties, + dbProperties sqlutil.DbProperties, serverName gomatrixserverlib.ServerName, serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, diff --git a/serverkeyapi/storage/keydb_wasm.go b/serverkeyapi/storage/keydb_wasm.go index 3d2ca5505..de66a1d63 100644 --- a/serverkeyapi/storage/keydb_wasm.go +++ b/serverkeyapi/storage/keydb_wasm.go @@ -22,7 +22,7 @@ import ( "golang.org/x/crypto/ed25519" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/serverkeyapi/storage/sqlite3" "github.com/matrix-org/gomatrixserverlib" ) @@ -30,7 +30,7 @@ import ( // NewDatabase opens a database connection. func NewDatabase( dataSourceName string, - dbProperties internal.DbProperties, // nolint:unparam + dbProperties sqlutil.DbProperties, // nolint:unparam serverName gomatrixserverlib.ServerName, serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, diff --git a/serverkeyapi/storage/postgres/keydb.go b/serverkeyapi/storage/postgres/keydb.go index 57a9a17e3..aaa4409be 100644 --- a/serverkeyapi/storage/postgres/keydb.go +++ b/serverkeyapi/storage/postgres/keydb.go @@ -20,7 +20,6 @@ import ( "golang.org/x/crypto/ed25519" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -37,7 +36,7 @@ type Database struct { // Returns an error if there was a problem talking to the database. func NewDatabase( dataSourceName string, - dbProperties internal.DbProperties, + dbProperties sqlutil.DbProperties, serverName gomatrixserverlib.ServerName, serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, diff --git a/serverkeyapi/storage/postgres/server_key_table.go b/serverkeyapi/storage/postgres/server_key_table.go index b3c26a480..87f1c211c 100644 --- a/serverkeyapi/storage/postgres/server_key_table.go +++ b/serverkeyapi/storage/postgres/server_key_table.go @@ -19,9 +19,8 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/internal" - "github.com/lib/pq" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/serverkeyapi/storage/sqlite3/keydb.go b/serverkeyapi/storage/sqlite3/keydb.go index a7a1db100..dc72b79eb 100644 --- a/serverkeyapi/storage/sqlite3/keydb.go +++ b/serverkeyapi/storage/sqlite3/keydb.go @@ -20,7 +20,6 @@ import ( "golang.org/x/crypto/ed25519" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" @@ -47,7 +46,7 @@ func NewDatabase( if err != nil { return nil, err } - db, err := sqlutil.Open(internal.SQLiteDriverName(), cs, nil) + db, err := sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil) if err != nil { return nil, err } diff --git a/serverkeyapi/storage/sqlite3/server_key_table.go b/serverkeyapi/storage/sqlite3/server_key_table.go index ae24a14d4..4f03dccbb 100644 --- a/serverkeyapi/storage/sqlite3/server_key_table.go +++ b/serverkeyapi/storage/sqlite3/server_key_table.go @@ -21,6 +21,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -90,7 +91,7 @@ func (s *serverKeyStatements) bulkSelectServerKeys( nameAndKeyIDs = append(nameAndKeyIDs, nameAndKeyID(request)) } - query := strings.Replace(bulkSelectServerKeysSQL, "($1)", internal.QueryVariadic(len(nameAndKeyIDs)), 1) + query := strings.Replace(bulkSelectServerKeysSQL, "($1)", sqlutil.QueryVariadic(len(nameAndKeyIDs)), 1) iKeyIDs := make([]interface{}, len(nameAndKeyIDs)) for i, v := range nameAndKeyIDs { diff --git a/syncapi/consumers/clientapi.go b/syncapi/consumers/clientapi.go index 4d43e8111..ad6290e3f 100644 --- a/syncapi/consumers/clientapi.go +++ b/syncapi/consumers/clientapi.go @@ -21,6 +21,7 @@ import ( "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/sync" "github.com/matrix-org/dendrite/syncapi/types" @@ -67,7 +68,7 @@ func (s *OutputClientDataConsumer) Start() error { // sync stream position may race and be incorrectly calculated. func (s *OutputClientDataConsumer) onMessage(msg *sarama.ConsumerMessage) error { // Parse out the event JSON - var output internal.AccountData + var output eventutil.AccountData if err := json.Unmarshal(msg.Value, &output); err != nil { // If the message was invalid, log it and move on to the next message in the stream log.WithError(err).Errorf("client API server output log: message parse failure") diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 055f76603..98be5bb73 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -17,7 +17,6 @@ package consumers import ( "context" "encoding/json" - "fmt" "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/internal" @@ -99,23 +98,10 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( ctx context.Context, msg api.OutputNewRoomEvent, ) error { ev := msg.Event - log.WithFields(log.Fields{ - "event_id": ev.EventID(), - "room_id": ev.RoomID(), - "room_version": ev.RoomVersion, - }).Info("received event from roomserver") - addsStateEvents, err := s.lookupStateEvents(msg.AddsStateEventIDs, ev) - if err != nil { - log.WithFields(log.Fields{ - "event": string(ev.JSON()), - log.ErrorKey: err, - "add": msg.AddsStateEventIDs, - "del": msg.RemovesStateEventIDs, - }).Panicf("roomserver output log: state event lookup failure") - } + addsStateEvents := msg.AddsState() - ev, err = s.updateStateEvent(ev) + ev, err := s.updateStateEvent(ev) if err != nil { return err } @@ -185,63 +171,6 @@ func (s *OutputRoomEventConsumer) onRetireInviteEvent( return nil } -// lookupStateEvents looks up the state events that are added by a new event. -func (s *OutputRoomEventConsumer) lookupStateEvents( - addsStateEventIDs []string, event gomatrixserverlib.HeaderedEvent, -) ([]gomatrixserverlib.HeaderedEvent, error) { - // Fast path if there aren't any new state events. - if len(addsStateEventIDs) == 0 { - return nil, nil - } - - // Fast path if the only state event added is the event itself. - if len(addsStateEventIDs) == 1 && addsStateEventIDs[0] == event.EventID() { - return []gomatrixserverlib.HeaderedEvent{event}, nil - } - - // Check if this is re-adding a state events that we previously processed - // If we have previously received a state event it may still be in - // our event database. - result, err := s.db.Events(context.TODO(), addsStateEventIDs) - if err != nil { - return nil, err - } - missing := missingEventsFrom(result, addsStateEventIDs) - - // Check if event itself is being added. - for _, eventID := range missing { - if eventID == event.EventID() { - result = append(result, event) - break - } - } - missing = missingEventsFrom(result, addsStateEventIDs) - - if len(missing) == 0 { - return result, nil - } - - // At this point the missing events are neither the event itself nor are - // they present in our local database. Our only option is to fetch them - // from the roomserver using the query API. - eventReq := api.QueryEventsByIDRequest{EventIDs: missing} - var eventResp api.QueryEventsByIDResponse - if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { - return nil, err - } - - result = append(result, eventResp.Events...) - missing = missingEventsFrom(result, addsStateEventIDs) - - if len(missing) != 0 { - return nil, fmt.Errorf( - "missing %d state events IDs at event %q", len(missing), event.EventID(), - ) - } - - return result, nil -} - func (s *OutputRoomEventConsumer) updateStateEvent(event gomatrixserverlib.HeaderedEvent) (gomatrixserverlib.HeaderedEvent, error) { var stateKey string if event.StateKey() == nil { @@ -270,17 +199,3 @@ func (s *OutputRoomEventConsumer) updateStateEvent(event gomatrixserverlib.Heade event.Event, err = event.SetUnsigned(prev) return event, err } - -func missingEventsFrom(events []gomatrixserverlib.HeaderedEvent, required []string) []string { - have := map[string]bool{} - for _, event := range events { - have[event.EventID()] = true - } - var missing []string - for _, eventID := range required { - if !have[eventID] { - missing = append(missing, eventID) - } - } - return missing -} diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 8c8976345..de5429db4 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -375,15 +375,15 @@ func (e eventsByDepth) Less(i, j int) bool { // Returns an error if there was an issue with retrieving the list of servers in // the room or sending the request. func (r *messagesReq) backfill(roomID string, backwardsExtremities map[string][]string, limit int) ([]gomatrixserverlib.HeaderedEvent, error) { - var res api.QueryBackfillResponse - err := r.rsAPI.QueryBackfill(context.Background(), &api.QueryBackfillRequest{ + var res api.PerformBackfillResponse + err := r.rsAPI.PerformBackfill(context.Background(), &api.PerformBackfillRequest{ RoomID: roomID, BackwardsExtremities: backwardsExtremities, Limit: limit, ServerName: r.cfg.Matrix.ServerName, }, &res) if err != nil { - return nil, fmt.Errorf("QueryBackfill failed: %w", err) + return nil, fmt.Errorf("PerformBackfill failed: %w", err) } util.GetLogger(r.ctx).WithField("new_events", len(res.Events)).Info("Storing new events from backfill") diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index c876164d7..50b469177 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -21,8 +21,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/sync" @@ -52,12 +52,12 @@ func Setup( } // TODO: Add AS support for all handlers below. - r0mux.Handle("/sync", internal.MakeAuthAPI("sync", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + r0mux.Handle("/sync", httputil.MakeAuthAPI("sync", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return srp.OnIncomingSyncRequest(req, device) })).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/messages", internal.MakeAuthAPI("room_messages", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + r0mux.Handle("/rooms/{roomID}/messages", httputil.MakeAuthAPI("room_messages", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } diff --git a/syncapi/storage/postgres/account_data_table.go b/syncapi/storage/postgres/account_data_table.go index e0b87bd6d..67eb1e863 100644 --- a/syncapi/storage/postgres/account_data_table.go +++ b/syncapi/storage/postgres/account_data_table.go @@ -21,6 +21,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -136,7 +137,7 @@ func (s *accountDataStatements) SelectMaxAccountDataID( ctx context.Context, txn *sql.Tx, ) (id int64, err error) { var nullableID sql.NullInt64 - stmt := internal.TxStmt(txn, s.selectMaxAccountDataIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectMaxAccountDataIDStmt) err = stmt.QueryRowContext(ctx).Scan(&nullableID) if nullableID.Valid { id = nullableID.Int64 diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index cc0fb6b1e..25906edb4 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -22,6 +22,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -165,7 +166,7 @@ func (s *currentRoomStateStatements) SelectRoomIDsWithMembership( userID string, membership string, // nolint: unparam ) ([]string, error) { - stmt := internal.TxStmt(txn, s.selectRoomIDsWithMembershipStmt) + stmt := sqlutil.TxStmt(txn, s.selectRoomIDsWithMembershipStmt) rows, err := stmt.QueryContext(ctx, userID, membership) if err != nil { return nil, err @@ -188,7 +189,7 @@ func (s *currentRoomStateStatements) SelectCurrentState( ctx context.Context, txn *sql.Tx, roomID string, stateFilter *gomatrixserverlib.StateFilter, ) ([]gomatrixserverlib.HeaderedEvent, error) { - stmt := internal.TxStmt(txn, s.selectCurrentStateStmt) + stmt := sqlutil.TxStmt(txn, s.selectCurrentStateStmt) rows, err := stmt.QueryContext(ctx, roomID, pq.StringArray(stateFilter.Senders), pq.StringArray(stateFilter.NotSenders), @@ -208,7 +209,7 @@ func (s *currentRoomStateStatements) SelectCurrentState( func (s *currentRoomStateStatements) DeleteRoomStateByEventID( ctx context.Context, txn *sql.Tx, eventID string, ) error { - stmt := internal.TxStmt(txn, s.deleteRoomStateByEventIDStmt) + stmt := sqlutil.TxStmt(txn, s.deleteRoomStateByEventIDStmt) _, err := stmt.ExecContext(ctx, eventID) return err } @@ -231,7 +232,7 @@ func (s *currentRoomStateStatements) UpsertRoomState( } // upsert state event - stmt := internal.TxStmt(txn, s.upsertRoomStateStmt) + stmt := sqlutil.TxStmt(txn, s.upsertRoomStateStmt) _, err = stmt.ExecContext( ctx, event.RoomID(), @@ -250,7 +251,7 @@ func (s *currentRoomStateStatements) UpsertRoomState( func (s *currentRoomStateStatements) SelectEventsWithEventIDs( ctx context.Context, txn *sql.Tx, eventIDs []string, ) ([]types.StreamEvent, error) { - stmt := internal.TxStmt(txn, s.selectEventsWithEventIDsStmt) + stmt := sqlutil.TxStmt(txn, s.selectEventsWithEventIDsStmt) rows, err := stmt.QueryContext(ctx, pq.StringArray(eventIDs)) if err != nil { return nil, err diff --git a/syncapi/storage/postgres/invites_table.go b/syncapi/storage/postgres/invites_table.go index caa12d2f8..5031d64e5 100644 --- a/syncapi/storage/postgres/invites_table.go +++ b/syncapi/storage/postgres/invites_table.go @@ -21,6 +21,7 @@ import ( "encoding/json" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -119,7 +120,7 @@ func (s *inviteEventsStatements) DeleteInviteEvent( func (s *inviteEventsStatements) SelectInviteEventsInRange( ctx context.Context, txn *sql.Tx, targetUserID string, r types.Range, ) (map[string]gomatrixserverlib.HeaderedEvent, error) { - stmt := internal.TxStmt(txn, s.selectInviteEventsInRangeStmt) + stmt := sqlutil.TxStmt(txn, s.selectInviteEventsInRangeStmt) rows, err := stmt.QueryContext(ctx, targetUserID, r.Low(), r.High()) if err != nil { return nil, err @@ -149,7 +150,7 @@ func (s *inviteEventsStatements) SelectMaxInviteID( ctx context.Context, txn *sql.Tx, ) (id int64, err error) { var nullableID sql.NullInt64 - stmt := internal.TxStmt(txn, s.selectMaxInviteIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectMaxInviteIDStmt) err = stmt.QueryRowContext(ctx).Scan(&nullableID) if nullableID.Valid { id = nullableID.Int64 diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index ae6dfb381..f01b2eabd 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -21,12 +21,13 @@ import ( "encoding/json" "sort" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/lib/pq" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" ) @@ -158,7 +159,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( ctx context.Context, txn *sql.Tx, r types.Range, stateFilter *gomatrixserverlib.StateFilter, ) (map[string]map[string]bool, map[string]types.StreamEvent, error) { - stmt := internal.TxStmt(txn, s.selectStateInRangeStmt) + stmt := sqlutil.TxStmt(txn, s.selectStateInRangeStmt) rows, err := stmt.QueryContext( ctx, r.Low(), r.High(), @@ -239,7 +240,7 @@ func (s *outputRoomEventsStatements) SelectMaxEventID( ctx context.Context, txn *sql.Tx, ) (id int64, err error) { var nullableID sql.NullInt64 - stmt := internal.TxStmt(txn, s.selectMaxEventIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectMaxEventIDStmt) err = stmt.QueryRowContext(ctx).Scan(&nullableID) if nullableID.Valid { id = nullableID.Int64 @@ -275,7 +276,7 @@ func (s *outputRoomEventsStatements) InsertEvent( return } - stmt := internal.TxStmt(txn, s.insertEventStmt) + stmt := sqlutil.TxStmt(txn, s.insertEventStmt) err = stmt.QueryRowContext( ctx, event.RoomID(), @@ -303,9 +304,9 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( ) ([]types.StreamEvent, error) { var stmt *sql.Stmt if onlySyncEvents { - stmt = internal.TxStmt(txn, s.selectRecentEventsForSyncStmt) + stmt = sqlutil.TxStmt(txn, s.selectRecentEventsForSyncStmt) } else { - stmt = internal.TxStmt(txn, s.selectRecentEventsStmt) + stmt = sqlutil.TxStmt(txn, s.selectRecentEventsStmt) } rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit) if err != nil { @@ -333,7 +334,7 @@ func (s *outputRoomEventsStatements) SelectEarlyEvents( ctx context.Context, txn *sql.Tx, roomID string, r types.Range, limit int, ) ([]types.StreamEvent, error) { - stmt := internal.TxStmt(txn, s.selectEarlyEventsStmt) + stmt := sqlutil.TxStmt(txn, s.selectEarlyEventsStmt) rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit) if err != nil { return nil, err @@ -357,7 +358,7 @@ func (s *outputRoomEventsStatements) SelectEarlyEvents( func (s *outputRoomEventsStatements) SelectEvents( ctx context.Context, txn *sql.Tx, eventIDs []string, ) ([]types.StreamEvent, error) { - stmt := internal.TxStmt(txn, s.selectEventsStmt) + stmt := sqlutil.TxStmt(txn, s.selectEventsStmt) rows, err := stmt.QueryContext(ctx, pq.StringArray(eventIDs)) if err != nil { return nil, err diff --git a/syncapi/storage/postgres/output_room_events_topology_table.go b/syncapi/storage/postgres/output_room_events_topology_table.go index ffbeece30..1ab3a1dc2 100644 --- a/syncapi/storage/postgres/output_room_events_topology_table.go +++ b/syncapi/storage/postgres/output_room_events_topology_table.go @@ -19,7 +19,6 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" diff --git a/syncapi/storage/postgres/send_to_device_table.go b/syncapi/storage/postgres/send_to_device_table.go index 335a05ef1..07af9ad6b 100644 --- a/syncapi/storage/postgres/send_to_device_table.go +++ b/syncapi/storage/postgres/send_to_device_table.go @@ -21,6 +21,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" ) @@ -107,14 +108,14 @@ func NewPostgresSendToDeviceTable(db *sql.DB) (tables.SendToDevice, error) { func (s *sendToDeviceStatements) InsertSendToDeviceMessage( ctx context.Context, txn *sql.Tx, userID, deviceID, content string, ) (err error) { - _, err = internal.TxStmt(txn, s.insertSendToDeviceMessageStmt).ExecContext(ctx, userID, deviceID, content) + _, err = sqlutil.TxStmt(txn, s.insertSendToDeviceMessageStmt).ExecContext(ctx, userID, deviceID, content) return } func (s *sendToDeviceStatements) CountSendToDeviceMessages( ctx context.Context, txn *sql.Tx, userID, deviceID string, ) (count int, err error) { - row := internal.TxStmt(txn, s.countSendToDeviceMessagesStmt).QueryRowContext(ctx, userID, deviceID) + row := sqlutil.TxStmt(txn, s.countSendToDeviceMessagesStmt).QueryRowContext(ctx, userID, deviceID) if err = row.Scan(&count); err != nil { return } @@ -124,7 +125,7 @@ func (s *sendToDeviceStatements) CountSendToDeviceMessages( func (s *sendToDeviceStatements) SelectSendToDeviceMessages( ctx context.Context, txn *sql.Tx, userID, deviceID string, ) (events []types.SendToDeviceEvent, err error) { - rows, err := internal.TxStmt(txn, s.selectSendToDeviceMessagesStmt).QueryContext(ctx, userID, deviceID) + rows, err := sqlutil.TxStmt(txn, s.selectSendToDeviceMessagesStmt).QueryContext(ctx, userID, deviceID) if err != nil { return } diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index 8a8f964a5..573586cc7 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -18,12 +18,10 @@ package postgres import ( "database/sql" - "github.com/matrix-org/dendrite/internal/sqlutil" - // Import the postgres database driver. _ "github.com/lib/pq" "github.com/matrix-org/dendrite/eduserver/cache" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/shared" ) @@ -32,11 +30,11 @@ import ( type SyncServerDatasource struct { shared.Database db *sql.DB - internal.PartitionOffsetStatements + sqlutil.PartitionOffsetStatements } // NewDatabase creates a new sync server database -func NewDatabase(dbDataSourceName string, dbProperties internal.DbProperties) (*SyncServerDatasource, error) { +func NewDatabase(dbDataSourceName string, dbProperties sqlutil.DbProperties) (*SyncServerDatasource, error) { var d SyncServerDatasource var err error if d.db, err = sqlutil.Open("postgres", dbDataSourceName, dbProperties); err != nil { @@ -82,7 +80,7 @@ func NewDatabase(dbDataSourceName string, dbProperties internal.DbProperties) (* CurrentRoomState: currState, BackwardExtremities: backwardExtremities, SendToDevice: sendToDevice, - SendToDeviceWriter: internal.NewTransactionWriter(), + SendToDeviceWriter: sqlutil.NewTransactionWriter(), EDUCache: cache.New(), } return &d, nil diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index 497c043a6..21d8df375 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -23,7 +23,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/eduserver/cache" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" @@ -42,7 +42,7 @@ type Database struct { CurrentRoomState tables.CurrentRoomState BackwardExtremities tables.BackwardsExtremities SendToDevice tables.SendToDevice - SendToDeviceWriter *internal.TransactionWriter + SendToDeviceWriter *sqlutil.TransactionWriter EDUCache *cache.EDUCache } @@ -126,7 +126,7 @@ func (d *Database) GetStateEvent( func (d *Database) GetStateEventsForRoom( ctx context.Context, roomID string, stateFilter *gomatrixserverlib.StateFilter, ) (stateEvents []gomatrixserverlib.HeaderedEvent, err error) { - err = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.DB, func(txn *sql.Tx) error { stateEvents, err = d.CurrentRoomState.SelectCurrentState(ctx, txn, roomID, stateFilter) return err }) @@ -136,7 +136,7 @@ func (d *Database) GetStateEventsForRoom( func (d *Database) SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) { var maxID int64 var err error - err = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.DB, func(txn *sql.Tx) error { maxID, err = d.OutputEvents.SelectMaxEventID(ctx, txn) if err != nil { return err @@ -168,7 +168,7 @@ func (d *Database) SyncStreamPosition(ctx context.Context) (types.StreamPosition func (d *Database) AddInviteEvent( ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, ) (sp types.StreamPosition, err error) { - err = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.DB, func(txn *sql.Tx) error { sp, err = d.Invites.InsertInviteEvent(ctx, txn, inviteEvent) return err }) @@ -207,7 +207,7 @@ func (d *Database) GetAccountDataInRange( func (d *Database) UpsertAccountData( ctx context.Context, userID, roomID, dataType string, ) (sp types.StreamPosition, err error) { - err = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.DB, func(txn *sql.Tx) error { sp, err = d.AccountData.InsertAccountData(ctx, txn, userID, roomID, dataType) return err }) @@ -275,7 +275,7 @@ func (d *Database) WriteEvent( addStateEventIDs, removeStateEventIDs []string, transactionID *api.TransactionID, excludeFromSync bool, ) (pduPosition types.StreamPosition, returnErr error) { - returnErr = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { + returnErr = sqlutil.WithTransaction(d.DB, func(txn *sql.Tx) error { var err error pos, err := d.OutputEvents.InsertEvent( ctx, txn, ev, addStateEventIDs, removeStateEventIDs, transactionID, excludeFromSync, @@ -375,7 +375,7 @@ func (d *Database) GetEventsInTopologicalRange( } func (d *Database) SyncPosition(ctx context.Context) (tok types.StreamingToken, err error) { - err = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.DB, func(txn *sql.Tx) error { pos, err := d.syncPositionTx(ctx, txn) if err != nil { return err @@ -454,7 +454,7 @@ func (d *Database) addPDUDeltaToResponse( } var succeeded bool defer func() { - txerr := internal.EndTransaction(txn, &succeeded) + txerr := sqlutil.EndTransaction(txn, &succeeded) if err == nil && txerr != nil { err = txerr } @@ -608,7 +608,7 @@ func (d *Database) getResponseWithPDUsForCompleteSync( } var succeeded bool defer func() { - txerr := internal.EndTransaction(txn, &succeeded) + txerr := sqlutil.EndTransaction(txn, &succeeded) if err == nil && txerr != nil { err = txerr } diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go index a3c797342..ae5caa4e5 100644 --- a/syncapi/storage/sqlite3/account_data_table.go +++ b/syncapi/storage/sqlite3/account_data_table.go @@ -20,7 +20,6 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index b0cf1297c..85f212ad8 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -22,6 +22,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -152,7 +153,7 @@ func (s *currentRoomStateStatements) SelectRoomIDsWithMembership( userID string, membership string, // nolint: unparam ) ([]string, error) { - stmt := internal.TxStmt(txn, s.selectRoomIDsWithMembershipStmt) + stmt := sqlutil.TxStmt(txn, s.selectRoomIDsWithMembershipStmt) rows, err := stmt.QueryContext(ctx, userID, membership) if err != nil { return nil, err @@ -175,7 +176,7 @@ func (s *currentRoomStateStatements) SelectCurrentState( ctx context.Context, txn *sql.Tx, roomID string, stateFilterPart *gomatrixserverlib.StateFilter, ) ([]gomatrixserverlib.HeaderedEvent, error) { - stmt := internal.TxStmt(txn, s.selectCurrentStateStmt) + stmt := sqlutil.TxStmt(txn, s.selectCurrentStateStmt) rows, err := stmt.QueryContext(ctx, roomID, nil, // FIXME: pq.StringArray(stateFilterPart.Senders), nil, // FIXME: pq.StringArray(stateFilterPart.NotSenders), @@ -195,7 +196,7 @@ func (s *currentRoomStateStatements) SelectCurrentState( func (s *currentRoomStateStatements) DeleteRoomStateByEventID( ctx context.Context, txn *sql.Tx, eventID string, ) error { - stmt := internal.TxStmt(txn, s.deleteRoomStateByEventIDStmt) + stmt := sqlutil.TxStmt(txn, s.deleteRoomStateByEventIDStmt) _, err := stmt.ExecContext(ctx, eventID) return err } @@ -218,7 +219,7 @@ func (s *currentRoomStateStatements) UpsertRoomState( } // upsert state event - stmt := internal.TxStmt(txn, s.upsertRoomStateStmt) + stmt := sqlutil.TxStmt(txn, s.upsertRoomStateStmt) _, err = stmt.ExecContext( ctx, event.RoomID(), @@ -241,7 +242,7 @@ func (s *currentRoomStateStatements) SelectEventsWithEventIDs( for k, v := range eventIDs { iEventIDs[k] = v } - query := strings.Replace(selectEventsWithEventIDsSQL, "($1)", internal.QueryVariadic(len(iEventIDs)), 1) + query := strings.Replace(selectEventsWithEventIDsSQL, "($1)", sqlutil.QueryVariadic(len(iEventIDs)), 1) rows, err := txn.QueryContext(ctx, query, iEventIDs...) if err != nil { return nil, err diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go index 64b523394..bb58e3456 100644 --- a/syncapi/storage/sqlite3/invites_table.go +++ b/syncapi/storage/sqlite3/invites_table.go @@ -21,6 +21,7 @@ import ( "encoding/json" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -123,7 +124,7 @@ func (s *inviteEventsStatements) DeleteInviteEvent( func (s *inviteEventsStatements) SelectInviteEventsInRange( ctx context.Context, txn *sql.Tx, targetUserID string, r types.Range, ) (map[string]gomatrixserverlib.HeaderedEvent, error) { - stmt := internal.TxStmt(txn, s.selectInviteEventsInRangeStmt) + stmt := sqlutil.TxStmt(txn, s.selectInviteEventsInRangeStmt) rows, err := stmt.QueryContext(ctx, targetUserID, r.Low(), r.High()) if err != nil { return nil, err @@ -153,7 +154,7 @@ func (s *inviteEventsStatements) SelectMaxInviteID( ctx context.Context, txn *sql.Tx, ) (id int64, err error) { var nullableID sql.NullInt64 - stmt := internal.TxStmt(txn, s.selectMaxInviteIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectMaxInviteIDStmt) err = stmt.QueryRowContext(ctx).Scan(&nullableID) if nullableID.Valid { id = nullableID.Int64 diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 4a84ecc6c..367ab3c9a 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -21,11 +21,12 @@ import ( "encoding/json" "sort" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" ) @@ -149,7 +150,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( ctx context.Context, txn *sql.Tx, r types.Range, stateFilterPart *gomatrixserverlib.StateFilter, ) (map[string]map[string]bool, map[string]types.StreamEvent, error) { - stmt := internal.TxStmt(txn, s.selectStateInRangeStmt) + stmt := sqlutil.TxStmt(txn, s.selectStateInRangeStmt) rows, err := stmt.QueryContext( ctx, r.Low(), r.High(), @@ -236,7 +237,7 @@ func (s *outputRoomEventsStatements) SelectMaxEventID( ctx context.Context, txn *sql.Tx, ) (id int64, err error) { var nullableID sql.NullInt64 - stmt := internal.TxStmt(txn, s.selectMaxEventIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectMaxEventIDStmt) err = stmt.QueryRowContext(ctx).Scan(&nullableID) if nullableID.Valid { id = nullableID.Int64 @@ -286,7 +287,7 @@ func (s *outputRoomEventsStatements) InsertEvent( return } - insertStmt := internal.TxStmt(txn, s.insertEventStmt) + insertStmt := sqlutil.TxStmt(txn, s.insertEventStmt) _, err = insertStmt.ExecContext( ctx, streamPos, @@ -313,9 +314,9 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( ) ([]types.StreamEvent, error) { var stmt *sql.Stmt if onlySyncEvents { - stmt = internal.TxStmt(txn, s.selectRecentEventsForSyncStmt) + stmt = sqlutil.TxStmt(txn, s.selectRecentEventsForSyncStmt) } else { - stmt = internal.TxStmt(txn, s.selectRecentEventsStmt) + stmt = sqlutil.TxStmt(txn, s.selectRecentEventsStmt) } rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit) @@ -342,7 +343,7 @@ func (s *outputRoomEventsStatements) SelectEarlyEvents( ctx context.Context, txn *sql.Tx, roomID string, r types.Range, limit int, ) ([]types.StreamEvent, error) { - stmt := internal.TxStmt(txn, s.selectEarlyEventsStmt) + stmt := sqlutil.TxStmt(txn, s.selectEarlyEventsStmt) rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit) if err != nil { return nil, err @@ -367,7 +368,7 @@ func (s *outputRoomEventsStatements) SelectEvents( ctx context.Context, txn *sql.Tx, eventIDs []string, ) ([]types.StreamEvent, error) { var returnEvents []types.StreamEvent - stmt := internal.TxStmt(txn, s.selectEventsStmt) + stmt := sqlutil.TxStmt(txn, s.selectEventsStmt) for _, eventID := range eventIDs { rows, err := stmt.QueryContext(ctx, eventID) if err != nil { diff --git a/syncapi/storage/sqlite3/output_room_events_topology_table.go b/syncapi/storage/sqlite3/output_room_events_topology_table.go index 0d727cb18..811dfa4f3 100644 --- a/syncapi/storage/sqlite3/output_room_events_topology_table.go +++ b/syncapi/storage/sqlite3/output_room_events_topology_table.go @@ -18,7 +18,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -102,7 +102,7 @@ func NewSqliteTopologyTable(db *sql.DB) (tables.Topology, error) { func (s *outputRoomEventsTopologyStatements) InsertEventInTopology( ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, pos types.StreamPosition, ) (err error) { - stmt := internal.TxStmt(txn, s.insertEventInTopologyStmt) + stmt := sqlutil.TxStmt(txn, s.insertEventInTopologyStmt) _, err = stmt.ExecContext( ctx, event.EventID(), event.Depth(), event.RoomID(), pos, ) @@ -118,9 +118,9 @@ func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( // is requested or not. var stmt *sql.Stmt if chronologicalOrder { - stmt = internal.TxStmt(txn, s.selectEventIDsInRangeASCStmt) + stmt = sqlutil.TxStmt(txn, s.selectEventIDsInRangeASCStmt) } else { - stmt = internal.TxStmt(txn, s.selectEventIDsInRangeDESCStmt) + stmt = sqlutil.TxStmt(txn, s.selectEventIDsInRangeDESCStmt) } // Query the event IDs. @@ -149,7 +149,7 @@ func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( func (s *outputRoomEventsTopologyStatements) SelectPositionInTopology( ctx context.Context, txn *sql.Tx, eventID string, ) (pos types.StreamPosition, spos types.StreamPosition, err error) { - stmt := internal.TxStmt(txn, s.selectPositionInTopologyStmt) + stmt := sqlutil.TxStmt(txn, s.selectPositionInTopologyStmt) err = stmt.QueryRowContext(ctx, eventID).Scan(&pos, &spos) return } @@ -157,7 +157,7 @@ func (s *outputRoomEventsTopologyStatements) SelectPositionInTopology( func (s *outputRoomEventsTopologyStatements) SelectMaxPositionInTopology( ctx context.Context, txn *sql.Tx, roomID string, ) (pos types.StreamPosition, spos types.StreamPosition, err error) { - stmt := internal.TxStmt(txn, s.selectMaxPositionInTopologyStmt) + stmt := sqlutil.TxStmt(txn, s.selectMaxPositionInTopologyStmt) err = stmt.QueryRowContext(ctx, roomID).Scan(&pos, &spos) return } diff --git a/syncapi/storage/sqlite3/send_to_device_table.go b/syncapi/storage/sqlite3/send_to_device_table.go index 0d03f23ef..42bd3c19a 100644 --- a/syncapi/storage/sqlite3/send_to_device_table.go +++ b/syncapi/storage/sqlite3/send_to_device_table.go @@ -21,6 +21,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" ) @@ -97,14 +98,14 @@ func NewSqliteSendToDeviceTable(db *sql.DB) (tables.SendToDevice, error) { func (s *sendToDeviceStatements) InsertSendToDeviceMessage( ctx context.Context, txn *sql.Tx, userID, deviceID, content string, ) (err error) { - _, err = internal.TxStmt(txn, s.insertSendToDeviceMessageStmt).ExecContext(ctx, userID, deviceID, content) + _, err = sqlutil.TxStmt(txn, s.insertSendToDeviceMessageStmt).ExecContext(ctx, userID, deviceID, content) return } func (s *sendToDeviceStatements) CountSendToDeviceMessages( ctx context.Context, txn *sql.Tx, userID, deviceID string, ) (count int, err error) { - row := internal.TxStmt(txn, s.countSendToDeviceMessagesStmt).QueryRowContext(ctx, userID, deviceID) + row := sqlutil.TxStmt(txn, s.countSendToDeviceMessagesStmt).QueryRowContext(ctx, userID, deviceID) if err = row.Scan(&count); err != nil { return } @@ -114,7 +115,7 @@ func (s *sendToDeviceStatements) CountSendToDeviceMessages( func (s *sendToDeviceStatements) SelectSendToDeviceMessages( ctx context.Context, txn *sql.Tx, userID, deviceID string, ) (events []types.SendToDeviceEvent, err error) { - rows, err := internal.TxStmt(txn, s.selectSendToDeviceMessagesStmt).QueryContext(ctx, userID, deviceID) + rows, err := sqlutil.TxStmt(txn, s.selectSendToDeviceMessagesStmt).QueryContext(ctx, userID, deviceID) if err != nil { return } @@ -149,7 +150,7 @@ func (s *sendToDeviceStatements) SelectSendToDeviceMessages( func (s *sendToDeviceStatements) UpdateSentSendToDeviceMessages( ctx context.Context, txn *sql.Tx, token string, nids []types.SendToDeviceNID, ) (err error) { - query := strings.Replace(updateSentSendToDeviceMessagesSQL, "($2)", internal.QueryVariadic(1+len(nids)), 1) + query := strings.Replace(updateSentSendToDeviceMessagesSQL, "($2)", sqlutil.QueryVariadic(1+len(nids)), 1) params := make([]interface{}, 1+len(nids)) params[0] = token for k, v := range nids { @@ -162,7 +163,7 @@ func (s *sendToDeviceStatements) UpdateSentSendToDeviceMessages( func (s *sendToDeviceStatements) DeleteSendToDeviceMessages( ctx context.Context, txn *sql.Tx, nids []types.SendToDeviceNID, ) (err error) { - query := strings.Replace(deleteSendToDeviceMessagesSQL, "($1)", internal.QueryVariadic(len(nids)), 1) + query := strings.Replace(deleteSendToDeviceMessagesSQL, "($1)", sqlutil.QueryVariadic(len(nids)), 1) params := make([]interface{}, 1+len(nids)) for k, v := range nids { params[k] = v diff --git a/syncapi/storage/sqlite3/stream_id_table.go b/syncapi/storage/sqlite3/stream_id_table.go index 3bfe22dd0..57abd9c44 100644 --- a/syncapi/storage/sqlite3/stream_id_table.go +++ b/syncapi/storage/sqlite3/stream_id_table.go @@ -4,7 +4,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/types" ) @@ -46,8 +46,8 @@ func (s *streamIDStatements) prepare(db *sql.DB) (err error) { } func (s *streamIDStatements) nextStreamID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { - increaseStmt := internal.TxStmt(txn, s.increaseStreamIDStmt) - selectStmt := internal.TxStmt(txn, s.selectStreamIDStmt) + increaseStmt := sqlutil.TxStmt(txn, s.increaseStreamIDStmt) + selectStmt := sqlutil.TxStmt(txn, s.selectStreamIDStmt) if _, err = increaseStmt.ExecContext(ctx, "global"); err != nil { return } diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 38ce5bcf0..51cdbe325 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -18,13 +18,11 @@ package sqlite3 import ( "database/sql" - "github.com/matrix-org/dendrite/internal/sqlutil" - // Import the sqlite3 package _ "github.com/mattn/go-sqlite3" "github.com/matrix-org/dendrite/eduserver/cache" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/shared" ) @@ -33,7 +31,7 @@ import ( type SyncServerDatasource struct { shared.Database db *sql.DB - internal.PartitionOffsetStatements + sqlutil.PartitionOffsetStatements streamID streamIDStatements } @@ -45,7 +43,7 @@ func NewDatabase(dataSourceName string) (*SyncServerDatasource, error) { if err != nil { return nil, err } - if d.db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil { + if d.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { return nil, err } if err = d.prepare(); err != nil { @@ -98,7 +96,7 @@ func (d *SyncServerDatasource) prepare() (err error) { CurrentRoomState: roomState, Topology: topology, SendToDevice: sendToDevice, - SendToDeviceWriter: internal.NewTransactionWriter(), + SendToDeviceWriter: sqlutil.NewTransactionWriter(), EDUCache: cache.New(), } return nil diff --git a/syncapi/storage/storage.go b/syncapi/storage/storage.go index a4275da0c..ea69da3bc 100644 --- a/syncapi/storage/storage.go +++ b/syncapi/storage/storage.go @@ -19,13 +19,13 @@ package storage import ( "net/url" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/postgres" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" ) // NewSyncServerDatasource opens a database connection. -func NewSyncServerDatasource(dataSourceName string, dbProperties internal.DbProperties) (Database, error) { +func NewSyncServerDatasource(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return postgres.NewDatabase(dataSourceName, dbProperties) diff --git a/syncapi/storage/storage_wasm.go b/syncapi/storage/storage_wasm.go index a2b269a14..0886b8c21 100644 --- a/syncapi/storage/storage_wasm.go +++ b/syncapi/storage/storage_wasm.go @@ -18,14 +18,14 @@ import ( "fmt" "net/url" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" ) // NewPublicRoomsServerDatabase opens a database connection. func NewSyncServerDatasource( dataSourceName string, - dbProperties internal.DbProperties, // nolint:unparam + dbProperties sqlutil.DbProperties, // nolint:unparam ) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { diff --git a/userapi/api/api.go b/userapi/api/api.go new file mode 100644 index 000000000..8534fb17e --- /dev/null +++ b/userapi/api/api.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import "context" + +// UserInternalAPI is the internal API for information about users and devices. +type UserInternalAPI interface { + QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error +} + +// QueryProfileRequest is the request for QueryProfile +type QueryProfileRequest struct { + // The user ID to query + UserID string +} + +// QueryProfileResponse is the response for QueryProfile +type QueryProfileResponse struct { + // True if the user has been created. Querying for a profile does not create them. + UserExists bool + // The current display name if set. + DisplayName string + // The current avatar URL if set. + AvatarURL string +} diff --git a/userapi/internal/api.go b/userapi/internal/api.go new file mode 100644 index 000000000..0144526c7 --- /dev/null +++ b/userapi/internal/api.go @@ -0,0 +1,53 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "context" + "database/sql" + "fmt" + + "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" + "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" + "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" +) + +type UserInternalAPI struct { + AccountDB accounts.Database + DeviceDB devices.Database + ServerName gomatrixserverlib.ServerName +} + +func (a *UserInternalAPI) QueryProfile(ctx context.Context, req *api.QueryProfileRequest, res *api.QueryProfileResponse) error { + local, domain, err := gomatrixserverlib.SplitID('@', req.UserID) + if err != nil { + return err + } + if domain != a.ServerName { + return fmt.Errorf("cannot query profile of remote users: got %s want %s", domain, a.ServerName) + } + prof, err := a.AccountDB.GetProfileByLocalpart(ctx, local) + if err != nil { + if err == sql.ErrNoRows { + return nil + } + return err + } + res.UserExists = true + res.AvatarURL = prof.AvatarURL + res.DisplayName = prof.DisplayName + return nil +} diff --git a/userapi/inthttp/client.go b/userapi/inthttp/client.go new file mode 100644 index 000000000..90cc54a48 --- /dev/null +++ b/userapi/inthttp/client.go @@ -0,0 +1,62 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inthttp + +import ( + "context" + "errors" + "net/http" + + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/userapi/api" + "github.com/opentracing/opentracing-go" +) + +// HTTP paths for the internal HTTP APIs +const ( + QueryProfilePath = "/userapi/queryProfile" +) + +// NewUserAPIClient creates a UserInternalAPI implemented by talking to a HTTP POST API. +// If httpClient is nil an error is returned +func NewUserAPIClient( + apiURL string, + httpClient *http.Client, +) (api.UserInternalAPI, error) { + if httpClient == nil { + return nil, errors.New("NewUserAPIClient: httpClient is ") + } + return &httpUserInternalAPI{ + apiURL: apiURL, + httpClient: httpClient, + }, nil +} + +type httpUserInternalAPI struct { + apiURL string + httpClient *http.Client +} + +func (h *httpUserInternalAPI) QueryProfile( + ctx context.Context, + request *api.QueryProfileRequest, + response *api.QueryProfileResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryProfile") + defer span.Finish() + + apiURL := h.apiURL + QueryProfilePath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} diff --git a/userapi/inthttp/server.go b/userapi/inthttp/server.go new file mode 100644 index 000000000..f3c17ccd4 --- /dev/null +++ b/userapi/inthttp/server.go @@ -0,0 +1,41 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inthttp + +import ( + "encoding/json" + "net/http" + + "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/util" +) + +func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) { + internalAPIMux.Handle(QueryProfilePath, + httputil.MakeInternalAPI("queryProfile", func(req *http.Request) util.JSONResponse { + request := api.QueryProfileRequest{} + response := api.QueryProfileResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := s.QueryProfile(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) +} diff --git a/userapi/userapi.go b/userapi/userapi.go new file mode 100644 index 000000000..32f851cc7 --- /dev/null +++ b/userapi/userapi.go @@ -0,0 +1,41 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package userapi + +import ( + "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" + "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" + "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/internal" + "github.com/matrix-org/dendrite/userapi/inthttp" + "github.com/matrix-org/gomatrixserverlib" +) + +// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions +// on the given input API. +func AddInternalRoutes(router *mux.Router, intAPI api.UserInternalAPI) { + inthttp.AddRoutes(router, intAPI) +} + +// NewInternalAPI returns a concerete implementation of the internal API. Callers +// can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. +func NewInternalAPI(accountDB accounts.Database, deviceDB devices.Database, serverName gomatrixserverlib.ServerName) api.UserInternalAPI { + return &internal.UserInternalAPI{ + AccountDB: accountDB, + DeviceDB: deviceDB, + ServerName: serverName, + } +} diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go new file mode 100644 index 000000000..423a86125 --- /dev/null +++ b/userapi/userapi_test.go @@ -0,0 +1,138 @@ +package userapi_test + +import ( + "context" + "fmt" + "net" + "net/http" + "reflect" + "sync" + "testing" + + "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" + "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/userapi" + "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/inthttp" + "github.com/matrix-org/gomatrixserverlib" +) + +const ( + serverName = gomatrixserverlib.ServerName("example.com") +) + +func MustMakeInternalAPI(t *testing.T) (api.UserInternalAPI, accounts.Database, devices.Database) { + accountDB, err := accounts.NewDatabase("file::memory:", nil, serverName) + if err != nil { + t.Fatalf("failed to create account DB: %s", err) + } + deviceDB, err := devices.NewDatabase("file::memory:", nil, serverName) + if err != nil { + t.Fatalf("failed to create device DB: %s", err) + } + + return userapi.NewInternalAPI(accountDB, deviceDB, serverName), accountDB, deviceDB +} + +func TestQueryProfile(t *testing.T) { + aliceAvatarURL := "mxc://example.com/alice" + aliceDisplayName := "Alice" + userAPI, accountDB, _ := MustMakeInternalAPI(t) + _, err := accountDB.CreateAccount(context.TODO(), "alice", "foobar", "") + if err != nil { + t.Fatalf("failed to make account: %s", err) + } + if err := accountDB.SetAvatarURL(context.TODO(), "alice", aliceAvatarURL); err != nil { + t.Fatalf("failed to set avatar url: %s", err) + } + if err := accountDB.SetDisplayName(context.TODO(), "alice", aliceDisplayName); err != nil { + t.Fatalf("failed to set display name: %s", err) + } + + testCases := []struct { + req api.QueryProfileRequest + wantRes api.QueryProfileResponse + wantErr error + }{ + { + req: api.QueryProfileRequest{ + UserID: fmt.Sprintf("@alice:%s", serverName), + }, + wantRes: api.QueryProfileResponse{ + UserExists: true, + AvatarURL: aliceAvatarURL, + DisplayName: aliceDisplayName, + }, + }, + { + req: api.QueryProfileRequest{ + UserID: fmt.Sprintf("@bob:%s", serverName), + }, + wantRes: api.QueryProfileResponse{ + UserExists: false, + }, + }, + { + req: api.QueryProfileRequest{ + UserID: "@alice:wrongdomain.com", + }, + wantErr: fmt.Errorf("wrong domain"), + }, + } + + runCases := func(testAPI api.UserInternalAPI) { + for _, tc := range testCases { + var gotRes api.QueryProfileResponse + gotErr := testAPI.QueryProfile(context.TODO(), &tc.req, &gotRes) + if tc.wantErr == nil && gotErr != nil || tc.wantErr != nil && gotErr == nil { + t.Errorf("QueryProfile error, got %s want %s", gotErr, tc.wantErr) + continue + } + if !reflect.DeepEqual(tc.wantRes, gotRes) { + t.Errorf("QueryProfile response got %+v want %+v", gotRes, tc.wantRes) + } + } + } + + t.Run("HTTP API", func(t *testing.T) { + router := mux.NewRouter().PathPrefix(httputil.InternalPathPrefix).Subrouter() + userapi.AddInternalRoutes(router, userAPI) + apiURL, cancel := listenAndServe(t, router) + defer cancel() + httpAPI, err := inthttp.NewUserAPIClient(apiURL, &http.Client{}) + if err != nil { + t.Fatalf("failed to create HTTP client") + } + runCases(httpAPI) + }) + t.Run("Monolith", func(t *testing.T) { + runCases(userAPI) + }) +} + +func listenAndServe(t *testing.T, router *mux.Router) (apiURL string, cancel func()) { + listener, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatalf("failed to listen: %s", err) + } + port := listener.Addr().(*net.TCPAddr).Port + srv := http.Server{} + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + srv.Handler = router + err := srv.Serve(listener) + if err != nil && err != http.ErrServerClosed { + t.Logf("Listen failed: %s", err) + } + }() + + return fmt.Sprintf("http://localhost:%d", port), func() { + srv.Shutdown(context.Background()) + wg.Wait() + } +}