From 35b7cbd5d8673d8ea82b36d90d9ad5b79dc3c5b7 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 28 Apr 2020 15:50:24 +0100 Subject: [PATCH 01/12] sql/backwards_extremities: Shift to table format and share code (#985) * sql/backwards_extremities: Shift to table format and share code This is an initial cut to reduce boilerplate at the storage layer. It removes the need for 2x `_table.go` files, one for each DB engine, replacing it with a single struct which has an interface which implements the raw SQL statements. The actual impl sits alongside the interface declaration which is generally regarded as best practice (though no canonical sources). Especially in this case where the impl is tiny (functions returning strings) and relies heavily on the function signatures of the table struct (for parameters), having the context in the same file is useful. * Remove _table redundancy --- .../postgres/backward_extremities_table.go | 124 ------------- syncapi/storage/postgres/syncserver.go | 18 +- .../sqlite3/backward_extremities_table.go | 124 ------------- syncapi/storage/sqlite3/syncserver.go | 18 +- .../storage/tables/backward_extremities.go | 175 ++++++++++++++++++ 5 files changed, 195 insertions(+), 264 deletions(-) delete mode 100644 syncapi/storage/postgres/backward_extremities_table.go delete mode 100644 syncapi/storage/sqlite3/backward_extremities_table.go create mode 100644 syncapi/storage/tables/backward_extremities.go diff --git a/syncapi/storage/postgres/backward_extremities_table.go b/syncapi/storage/postgres/backward_extremities_table.go deleted file mode 100644 index cb3629644..000000000 --- a/syncapi/storage/postgres/backward_extremities_table.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2018 New Vector Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package postgres - -import ( - "context" - "database/sql" - - "github.com/matrix-org/dendrite/common" -) - -// The purpose of this table is to keep track of backwards extremities for a room. -// Backwards extremities are the earliest (DAG-wise) known events which we have -// the entire event JSON. These event IDs are used in federation requests to fetch -// even earlier events. -// -// We persist the previous event IDs as well, one per row, so when we do fetch even -// earlier events we can simply delete rows which referenced it. Consider the graph: -// A -// | Event C has 1 prev_event ID: A. -// B C -// |___| Event D has 2 prev_event IDs: B and C. -// | -// D -// The earliest known event we have is D, so this table has 2 rows. -// A backfill request gives us C but not B. We delete rows where prev_event=C. This -// still means that D is a backwards extremity as we do not have event B. However, event -// C is *also* a backwards extremity at this point as we do not have event A. Later, -// when we fetch event B, we delete rows where prev_event=B. This then removes D as -// a backwards extremity because there are no more rows with event_id=B. -const backwardExtremitiesSchema = ` --- Stores output room events received from the roomserver. -CREATE TABLE IF NOT EXISTS syncapi_backward_extremities ( - -- The 'room_id' key for the event. - room_id TEXT NOT NULL, - -- The event ID for the last known event. This is the backwards extremity. - event_id TEXT NOT NULL, - -- The prev_events for the last known event. This is used to update extremities. - prev_event_id TEXT NOT NULL, - - PRIMARY KEY(room_id, event_id, prev_event_id) -); -` - -const insertBackwardExtremitySQL = "" + - "INSERT INTO syncapi_backward_extremities (room_id, event_id, prev_event_id)" + - " VALUES ($1, $2, $3)" + - " ON CONFLICT DO NOTHING" - -const selectBackwardExtremitiesForRoomSQL = "" + - "SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1" - -const deleteBackwardExtremitySQL = "" + - "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" - -type backwardExtremitiesStatements struct { - insertBackwardExtremityStmt *sql.Stmt - selectBackwardExtremitiesForRoomStmt *sql.Stmt - deleteBackwardExtremityStmt *sql.Stmt -} - -func (s *backwardExtremitiesStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(backwardExtremitiesSchema) - if err != nil { - return - } - if s.insertBackwardExtremityStmt, err = db.Prepare(insertBackwardExtremitySQL); err != nil { - return - } - if s.selectBackwardExtremitiesForRoomStmt, err = db.Prepare(selectBackwardExtremitiesForRoomSQL); err != nil { - return - } - if s.deleteBackwardExtremityStmt, err = db.Prepare(deleteBackwardExtremitySQL); err != nil { - return - } - return -} - -func (s *backwardExtremitiesStatements) insertsBackwardExtremity( - ctx context.Context, txn *sql.Tx, roomID, eventID string, prevEventID string, -) (err error) { - _, err = txn.Stmt(s.insertBackwardExtremityStmt).ExecContext(ctx, roomID, eventID, prevEventID) - return -} - -func (s *backwardExtremitiesStatements) selectBackwardExtremitiesForRoom( - ctx context.Context, roomID string, -) (eventIDs []string, err error) { - rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID) - if err != nil { - return - } - defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") - - for rows.Next() { - var eID string - if err = rows.Scan(&eID); err != nil { - return - } - - eventIDs = append(eventIDs, eID) - } - - return eventIDs, rows.Err() -} - -func (s *backwardExtremitiesStatements) deleteBackwardExtremity( - ctx context.Context, txn *sql.Tx, roomID, knownEventID string, -) (err error) { - _, err = txn.Stmt(s.deleteBackwardExtremityStmt).ExecContext(ctx, roomID, knownEventID) - return -} diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index 1e078ef46..9d61ccfc3 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -32,6 +32,7 @@ import ( _ "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -56,7 +57,7 @@ type SyncServerDatasource struct { invites inviteEventsStatements eduCache *cache.EDUCache topology outputRoomEventsTopologyStatements - backwardExtremities backwardExtremitiesStatements + backwardExtremities tables.BackwardsExtremities } // NewSyncServerDatasource creates a new sync server database @@ -75,16 +76,17 @@ func NewSyncServerDatasource(dbDataSourceName string) (*SyncServerDatasource, er if err = d.events.prepare(d.db); err != nil { return nil, err } - if err := d.roomstate.prepare(d.db); err != nil { + if err = d.roomstate.prepare(d.db); err != nil { return nil, err } - if err := d.invites.prepare(d.db); err != nil { + if err = d.invites.prepare(d.db); err != nil { return nil, err } - if err := d.topology.prepare(d.db); err != nil { + if err = d.topology.prepare(d.db); err != nil { return nil, err } - if err := d.backwardExtremities.prepare(d.db); err != nil { + d.backwardExtremities, err = tables.NewBackwardsExtremities(d.db, &tables.PostgresBackwardsExtremitiesStatements{}) + if err != nil { return nil, err } d.eduCache = cache.New() @@ -116,7 +118,7 @@ func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([ // the events listed in the event's 'prev_events'. This function also updates the backwards extremities table // to account for the fact that the given event is no longer a backwards extremity, but may be marked as such. func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *gomatrixserverlib.HeaderedEvent) error { - if err := d.backwardExtremities.deleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { + if err := d.backwardExtremities.DeleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { return err } @@ -137,7 +139,7 @@ func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, tx // If the event is missing, consider it a backward extremity. if !found { - if err = d.backwardExtremities.insertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil { + if err = d.backwardExtremities.InsertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil { return err } } @@ -314,7 +316,7 @@ func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (types.Paginati func (d *SyncServerDatasource) BackwardExtremitiesForRoom( ctx context.Context, roomID string, ) (backwardExtremities []string, err error) { - return d.backwardExtremities.selectBackwardExtremitiesForRoom(ctx, roomID) + return d.backwardExtremities.SelectBackwardExtremitiesForRoom(ctx, roomID) } // MaxTopologicalPosition returns the highest topological position for a given diff --git a/syncapi/storage/sqlite3/backward_extremities_table.go b/syncapi/storage/sqlite3/backward_extremities_table.go deleted file mode 100644 index 3d8cb91fc..000000000 --- a/syncapi/storage/sqlite3/backward_extremities_table.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2018 New Vector Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sqlite3 - -import ( - "context" - "database/sql" - - "github.com/matrix-org/dendrite/common" -) - -// The purpose of this table is to keep track of backwards extremities for a room. -// Backwards extremities are the earliest (DAG-wise) known events which we have -// the entire event JSON. These event IDs are used in federation requests to fetch -// even earlier events. -// -// We persist the previous event IDs as well, one per row, so when we do fetch even -// earlier events we can simply delete rows which referenced it. Consider the graph: -// A -// | Event C has 1 prev_event ID: A. -// B C -// |___| Event D has 2 prev_event IDs: B and C. -// | -// D -// The earliest known event we have is D, so this table has 2 rows. -// A backfill request gives us C but not B. We delete rows where prev_event=C. This -// still means that D is a backwards extremity as we do not have event B. However, event -// C is *also* a backwards extremity at this point as we do not have event A. Later, -// when we fetch event B, we delete rows where prev_event=B. This then removes D as -// a backwards extremity because there are no more rows with event_id=B. -const backwardExtremitiesSchema = ` --- Stores output room events received from the roomserver. -CREATE TABLE IF NOT EXISTS syncapi_backward_extremities ( - -- The 'room_id' key for the event. - room_id TEXT NOT NULL, - -- The event ID for the last known event. This is the backwards extremity. - event_id TEXT NOT NULL, - -- The prev_events for the last known event. This is used to update extremities. - prev_event_id TEXT NOT NULL, - - PRIMARY KEY(room_id, event_id, prev_event_id) -); -` - -const insertBackwardExtremitySQL = "" + - "INSERT INTO syncapi_backward_extremities (room_id, event_id, prev_event_id)" + - " VALUES ($1, $2, $3)" + - " ON CONFLICT (room_id, event_id, prev_event_id) DO NOTHING" - -const selectBackwardExtremitiesForRoomSQL = "" + - "SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1" - -const deleteBackwardExtremitySQL = "" + - "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" - -type backwardExtremitiesStatements struct { - insertBackwardExtremityStmt *sql.Stmt - selectBackwardExtremitiesForRoomStmt *sql.Stmt - deleteBackwardExtremityStmt *sql.Stmt -} - -func (s *backwardExtremitiesStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(backwardExtremitiesSchema) - if err != nil { - return - } - if s.insertBackwardExtremityStmt, err = db.Prepare(insertBackwardExtremitySQL); err != nil { - return - } - if s.selectBackwardExtremitiesForRoomStmt, err = db.Prepare(selectBackwardExtremitiesForRoomSQL); err != nil { - return - } - if s.deleteBackwardExtremityStmt, err = db.Prepare(deleteBackwardExtremitySQL); err != nil { - return - } - return -} - -func (s *backwardExtremitiesStatements) insertsBackwardExtremity( - ctx context.Context, txn *sql.Tx, roomID, eventID string, prevEventID string, -) (err error) { - _, err = txn.Stmt(s.insertBackwardExtremityStmt).ExecContext(ctx, roomID, eventID, prevEventID) - return -} - -func (s *backwardExtremitiesStatements) selectBackwardExtremitiesForRoom( - ctx context.Context, roomID string, -) (eventIDs []string, err error) { - rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID) - if err != nil { - return - } - defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") - - for rows.Next() { - var eID string - if err = rows.Scan(&eID); err != nil { - return - } - - eventIDs = append(eventIDs, eID) - } - - return eventIDs, rows.Err() -} - -func (s *backwardExtremitiesStatements) deleteBackwardExtremity( - ctx context.Context, txn *sql.Tx, roomID, knownEventID string, -) (err error) { - _, err = txn.Stmt(s.deleteBackwardExtremityStmt).ExecContext(ctx, roomID, knownEventID) - return -} diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index cdfd29b84..35774830e 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -35,6 +35,7 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -60,7 +61,7 @@ type SyncServerDatasource struct { invites inviteEventsStatements eduCache *cache.EDUCache topology outputRoomEventsTopologyStatements - backwardExtremities backwardExtremitiesStatements + backwardExtremities tables.BackwardsExtremities } // NewSyncServerDatasource creates a new sync server database @@ -102,16 +103,17 @@ func (d *SyncServerDatasource) prepare() (err error) { if err = d.events.prepare(d.db, &d.streamID); err != nil { return err } - if err := d.roomstate.prepare(d.db, &d.streamID); err != nil { + if err = d.roomstate.prepare(d.db, &d.streamID); err != nil { return err } - if err := d.invites.prepare(d.db, &d.streamID); err != nil { + if err = d.invites.prepare(d.db, &d.streamID); err != nil { return err } - if err := d.topology.prepare(d.db); err != nil { + if err = d.topology.prepare(d.db); err != nil { return err } - if err := d.backwardExtremities.prepare(d.db); err != nil { + d.backwardExtremities, err = tables.NewBackwardsExtremities(d.db, &tables.SqliteBackwardsExtremitiesStatements{}) + if err != nil { return err } return nil @@ -142,7 +144,7 @@ func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([ // the events listed in the event's 'prev_events'. This function also updates the backwards extremities table // to account for the fact that the given event is no longer a backwards extremity, but may be marked as such. func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *gomatrixserverlib.HeaderedEvent) error { - if err := d.backwardExtremities.deleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { + if err := d.backwardExtremities.DeleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { return err } @@ -163,7 +165,7 @@ func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, tx // If the event is missing, consider it a backward extremity. if !found { - if err = d.backwardExtremities.insertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil { + if err = d.backwardExtremities.InsertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil { return err } } @@ -344,7 +346,7 @@ func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (tok types.Pagi func (d *SyncServerDatasource) BackwardExtremitiesForRoom( ctx context.Context, roomID string, ) (backwardExtremities []string, err error) { - return d.backwardExtremities.selectBackwardExtremitiesForRoom(ctx, roomID) + return d.backwardExtremities.SelectBackwardExtremitiesForRoom(ctx, roomID) } // MaxTopologicalPosition returns the highest topological position for a given diff --git a/syncapi/storage/tables/backward_extremities.go b/syncapi/storage/tables/backward_extremities.go new file mode 100644 index 000000000..babd9aaa1 --- /dev/null +++ b/syncapi/storage/tables/backward_extremities.go @@ -0,0 +1,175 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tables + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/common" +) + +// BackwardsExtremitiesStatements contains the SQL statements to implement. +// See BackwardsExtremities to see the parameter and response types. +type BackwardsExtremitiesStatements interface { + Schema() string + InsertBackwardExtremity() string + SelectBackwardExtremitiesForRoom() string + DeleteBackwardExtremity() string +} + +type PostgresBackwardsExtremitiesStatements struct{} + +func (s *PostgresBackwardsExtremitiesStatements) Schema() string { + return `-- Stores output room events received from the roomserver. + CREATE TABLE IF NOT EXISTS syncapi_backward_extremities ( + -- The 'room_id' key for the event. + room_id TEXT NOT NULL, + -- The event ID for the last known event. This is the backwards extremity. + event_id TEXT NOT NULL, + -- The prev_events for the last known event. This is used to update extremities. + prev_event_id TEXT NOT NULL, + + PRIMARY KEY(room_id, event_id, prev_event_id) + ); + ` +} +func (s *PostgresBackwardsExtremitiesStatements) InsertBackwardExtremity() string { + return "" + + "INSERT INTO syncapi_backward_extremities (room_id, event_id, prev_event_id)" + + " VALUES ($1, $2, $3)" + + " ON CONFLICT DO NOTHING" +} +func (s *PostgresBackwardsExtremitiesStatements) SelectBackwardExtremitiesForRoom() string { + return "SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1" +} +func (s *PostgresBackwardsExtremitiesStatements) DeleteBackwardExtremity() string { + return "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" +} + +type SqliteBackwardsExtremitiesStatements struct{} + +func (s *SqliteBackwardsExtremitiesStatements) Schema() string { + return `-- Stores output room events received from the roomserver. +CREATE TABLE IF NOT EXISTS syncapi_backward_extremities ( + -- The 'room_id' key for the event. + room_id TEXT NOT NULL, + -- The event ID for the last known event. This is the backwards extremity. + event_id TEXT NOT NULL, + -- The prev_events for the last known event. This is used to update extremities. + prev_event_id TEXT NOT NULL, + + PRIMARY KEY(room_id, event_id, prev_event_id) +); +` +} + +func (s *SqliteBackwardsExtremitiesStatements) InsertBackwardExtremity() string { + return "" + + "INSERT INTO syncapi_backward_extremities (room_id, event_id, prev_event_id)" + + " VALUES ($1, $2, $3)" + + " ON CONFLICT (room_id, event_id, prev_event_id) DO NOTHING" +} + +func (s *SqliteBackwardsExtremitiesStatements) SelectBackwardExtremitiesForRoom() string { + return "" + + "SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1" +} + +func (s *SqliteBackwardsExtremitiesStatements) DeleteBackwardExtremity() string { + return "" + + "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" +} + +// BackwardsExtremities keeps track of backwards extremities for a room. +// Backwards extremities are the earliest (DAG-wise) known events which we have +// the entire event JSON. These event IDs are used in federation requests to fetch +// even earlier events. +// +// We persist the previous event IDs as well, one per row, so when we do fetch even +// earlier events we can simply delete rows which referenced it. Consider the graph: +// A +// | Event C has 1 prev_event ID: A. +// B C +// |___| Event D has 2 prev_event IDs: B and C. +// | +// D +// The earliest known event we have is D, so this table has 2 rows. +// A backfill request gives us C but not B. We delete rows where prev_event=C. This +// still means that D is a backwards extremity as we do not have event B. However, event +// C is *also* a backwards extremity at this point as we do not have event A. Later, +// when we fetch event B, we delete rows where prev_event=B. This then removes D as +// a backwards extremity because there are no more rows with event_id=B. +type BackwardsExtremities struct { + insertBackwardExtremityStmt *sql.Stmt + selectBackwardExtremitiesForRoomStmt *sql.Stmt + deleteBackwardExtremityStmt *sql.Stmt +} + +// NewBackwardsExtremities prepares the table +func NewBackwardsExtremities(db *sql.DB, stmts BackwardsExtremitiesStatements) (table BackwardsExtremities, err error) { + _, err = db.Exec(stmts.Schema()) + if err != nil { + return + } + if table.insertBackwardExtremityStmt, err = db.Prepare(stmts.InsertBackwardExtremity()); err != nil { + return + } + if table.selectBackwardExtremitiesForRoomStmt, err = db.Prepare(stmts.SelectBackwardExtremitiesForRoom()); err != nil { + return + } + if table.deleteBackwardExtremityStmt, err = db.Prepare(stmts.DeleteBackwardExtremity()); err != nil { + return + } + return +} + +// InsertsBackwardExtremity inserts a new backwards extremity. +func (s *BackwardsExtremities) InsertsBackwardExtremity( + ctx context.Context, txn *sql.Tx, roomID, eventID string, prevEventID string, +) (err error) { + _, err = txn.Stmt(s.insertBackwardExtremityStmt).ExecContext(ctx, roomID, eventID, prevEventID) + return +} + +// SelectBackwardExtremitiesForRoom retrieves all backwards extremities for the room. +func (s *BackwardsExtremities) SelectBackwardExtremitiesForRoom( + ctx context.Context, roomID string, +) (eventIDs []string, err error) { + rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID) + if err != nil { + return + } + defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") + + for rows.Next() { + var eID string + if err = rows.Scan(&eID); err != nil { + return + } + + eventIDs = append(eventIDs, eID) + } + + return eventIDs, rows.Err() +} + +// DeleteBackwardExtremity removes a backwards extremity for a room, if one existed. +func (s *BackwardsExtremities) DeleteBackwardExtremity( + ctx context.Context, txn *sql.Tx, roomID, knownEventID string, +) (err error) { + _, err = txn.Stmt(s.deleteBackwardExtremityStmt).ExecContext(ctx, roomID, knownEventID) + return +} From 0354836b57aa0f1e39c43961669ab1afcb757460 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 28 Apr 2020 16:22:00 +0100 Subject: [PATCH 02/12] Unbreak the wasm build (#986) --- build.sh | 4 +++- cmd/dendritejs/main.go | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/build.sh b/build.sh index 3ef148891..087f4ae72 100755 --- a/build.sh +++ b/build.sh @@ -3,4 +3,6 @@ # Put installed packages into ./bin export GOBIN=$PWD/`dirname $0`/bin -go install -v $PWD/`dirname $0`/cmd/... \ No newline at end of file +go install -v $PWD/`dirname $0`/cmd/... + +GOOS=js GOARCH=wasm go build -o main.wasm ./cmd/dendritejs diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 9bd8f2ee2..23d4327d7 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -123,12 +123,12 @@ func main() { } p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node) - alias, input, query := roomserver.SetupRoomServerComponent(base) + alias, input, query := roomserver.SetupRoomServerComponent(base, keyRing) eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) asQuery := appservice.SetupAppServiceAPIComponent( base, accountDB, deviceDB, federation, alias, query, transactions.New(), ) - fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query) + fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query, input) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, From a4b9edb28e32b505cf3a67bcba1acacd4a882155 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 28 Apr 2020 16:51:16 +0100 Subject: [PATCH 03/12] Dependency inject the federation client so p2p binaries work as expected (#987) --- cmd/dendrite-demo-libp2p/main.go | 2 +- cmd/dendrite-monolith-server/main.go | 2 +- cmd/dendrite-room-server/main.go | 2 +- cmd/dendritejs/main.go | 2 +- roomserver/roomserver.go | 3 ++- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index f1d18d0e0..66d103eaf 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -148,7 +148,7 @@ func main() { federation := createFederationClient(base) keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - alias, input, query := roomserver.SetupRoomServerComponent(&base.Base, keyRing) + alias, input, query := roomserver.SetupRoomServerComponent(&base.Base, keyRing, federation) eduInputAPI := eduserver.SetupEDUServerComponent(&base.Base, cache.New()) asQuery := appservice.SetupAppServiceAPIComponent( &base.Base, accountDB, deviceDB, federation, alias, query, transactions.New(), diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 8cfb75665..f3e720a88 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -57,7 +57,7 @@ func main() { federation := base.CreateFederationClient() keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - alias, input, query := roomserver.SetupRoomServerComponent(base, keyRing) + alias, input, query := roomserver.SetupRoomServerComponent(base, keyRing, federation) eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) asQuery := appservice.SetupAppServiceAPIComponent( base, accountDB, deviceDB, federation, alias, query, transactions.New(), diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go index f33a2b88f..98410452f 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-room-server/main.go @@ -30,7 +30,7 @@ func main() { federation := base.CreateFederationClient() keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - roomserver.SetupRoomServerComponent(base, keyRing) + roomserver.SetupRoomServerComponent(base, keyRing, federation) base.SetupAndServeHTTP(string(base.Cfg.Bind.RoomServer), string(base.Cfg.Listen.RoomServer)) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 23d4327d7..6300d249f 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -123,7 +123,7 @@ func main() { } p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node) - alias, input, query := roomserver.SetupRoomServerComponent(base, keyRing) + alias, input, query := roomserver.SetupRoomServerComponent(base, keyRing, federation) eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) asQuery := appservice.SetupAppServiceAPIComponent( base, accountDB, deviceDB, federation, alias, query, transactions.New(), diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index ea1c5c4c3..f6a373a4f 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -35,6 +35,7 @@ import ( // APIs directly instead of having to use HTTP. func SetupRoomServerComponent( base *basecomponent.BaseDendrite, keyRing gomatrixserverlib.JSONVerifier, + fedClient *gomatrixserverlib.FederationClient, ) (api.RoomserverAliasAPI, api.RoomserverInputAPI, api.RoomserverQueryAPI) { roomserverDB, err := storage.Open(string(base.Cfg.Database.RoomServer)) if err != nil { @@ -53,7 +54,7 @@ func SetupRoomServerComponent( DB: roomserverDB, ImmutableCache: base.ImmutableCache, ServerName: base.Cfg.Matrix.ServerName, - FedClient: base.CreateFederationClient(), + FedClient: fedClient, // TODO: We should have a key server so we don't keep adding components // which talk to the same DB. KeyRing: keyRing, From a308e61331f549ae0964f83dff88abc282033ed3 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 29 Apr 2020 11:34:31 +0100 Subject: [PATCH 04/12] Federation sender API remodel (#988) * Define an input API for the federationsender * Wiring for rooomserver input API and federation sender input API * Whoops, commit common too * Merge input API into query API * Rename FederationSenderQueryAPI to FederationSenderInternalAPI * Fix dendritejs * Rename Input to Perform * Fix a couple of inputs -> performs * Remove needless storage interface, add comments --- clientapi/clientapi.go | 4 +- clientapi/routing/directory.go | 2 +- clientapi/routing/routing.go | 2 +- cmd/dendrite-client-api-server/main.go | 4 +- cmd/dendrite-demo-libp2p/main.go | 6 +- cmd/dendrite-federation-api-server/main.go | 4 +- cmd/dendrite-monolith-server/main.go | 7 +- cmd/dendrite-room-server/main.go | 4 +- cmd/dendritejs/main.go | 1 + common/basecomponent/base.go | 8 +-- federationapi/federationapi.go | 2 +- federationapi/routing/query.go | 2 +- federationapi/routing/routing.go | 2 +- federationsender/api/api.go | 53 ++++++++++++++ federationsender/api/perform.go | 56 +++++++++++++++ federationsender/api/query.go | 80 ++++++---------------- federationsender/federationsender.go | 4 +- federationsender/query/api.go | 77 +++++++++++++++++++++ federationsender/query/perform.go | 25 +++++++ federationsender/query/query.go | 57 ++------------- roomserver/api/input.go | 16 ++++- roomserver/input/input.go | 12 ++++ roomserver/roomserver.go | 1 + 23 files changed, 293 insertions(+), 136 deletions(-) create mode 100644 federationsender/api/api.go create mode 100644 federationsender/api/perform.go create mode 100644 federationsender/query/api.go create mode 100644 federationsender/query/perform.go diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 1339f7c8c..6a857e52c 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -44,7 +44,7 @@ func SetupClientAPIComponent( eduInputAPI eduServerAPI.EDUServerInputAPI, asAPI appserviceAPI.AppServiceQueryAPI, transactionsCache *transactions.Cache, - fedSenderAPI federationSenderAPI.FederationSenderQueryAPI, + fsAPI federationSenderAPI.FederationSenderInternalAPI, ) { roomserverProducer := producers.NewRoomserverProducer(inputAPI, queryAPI) eduProducer := producers.NewEDUServerProducer(eduInputAPI) @@ -69,6 +69,6 @@ func SetupClientAPIComponent( routing.Setup( base.APIMux, base.Cfg, roomserverProducer, queryAPI, aliasAPI, asAPI, accountsDB, deviceDB, federation, *keyRing, userUpdateProducer, - syncProducer, eduProducer, transactionsCache, fedSenderAPI, + syncProducer, eduProducer, transactionsCache, fsAPI, ) } diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index 248696ab2..101ba11ff 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -47,7 +47,7 @@ func DirectoryRoom( federation *gomatrixserverlib.FederationClient, cfg *config.Dendrite, rsAPI roomserverAPI.RoomserverAliasAPI, - fedSenderAPI federationSenderAPI.FederationSenderQueryAPI, + fedSenderAPI federationSenderAPI.FederationSenderInternalAPI, ) util.JSONResponse { _, domain, err := gomatrixserverlib.SplitID('#', roomAlias) if err != nil { diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 5dc6d7db9..9ab22cbec 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -60,7 +60,7 @@ func Setup( syncProducer *producers.SyncAPIProducer, eduProducer *producers.EDUServerProducer, transactionsCache *transactions.Cache, - federationSender federationSenderAPI.FederationSenderQueryAPI, + federationSender federationSenderAPI.FederationSenderInternalAPI, ) { apiMux.Handle("/_matrix/client/versions", diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index 815a978a8..c8f629689 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -37,12 +37,12 @@ func main() { asQuery := base.CreateHTTPAppServiceAPIs() alias, input, query := base.CreateHTTPRoomserverAPIs() - fedSenderAPI := base.CreateHTTPFederationSenderAPIs() + fsAPI := base.CreateHTTPFederationSenderAPIs() eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, federation, &keyRing, - alias, input, query, eduInputAPI, asQuery, transactions.New(), fedSenderAPI, + alias, input, query, eduInputAPI, asQuery, transactions.New(), fsAPI, ) base.SetupAndServeHTTP(string(base.Cfg.Bind.ClientAPI), string(base.Cfg.Listen.ClientAPI)) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 66d103eaf..b9fbfc53e 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -153,15 +153,15 @@ func main() { asQuery := appservice.SetupAppServiceAPIComponent( &base.Base, accountDB, deviceDB, federation, alias, query, transactions.New(), ) - fedSenderAPI := federationsender.SetupFederationSenderComponent(&base.Base, federation, query, input) + fsAPI := federationsender.SetupFederationSenderComponent(&base.Base, federation, query, input) clientapi.SetupClientAPIComponent( &base.Base, deviceDB, accountDB, federation, &keyRing, alias, input, query, - eduInputAPI, asQuery, transactions.New(), fedSenderAPI, + eduInputAPI, asQuery, transactions.New(), fsAPI, ) eduProducer := producers.NewEDUServerProducer(eduInputAPI) - federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer) + federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fsAPI, eduProducer) mediaapi.SetupMediaAPIComponent(&base.Base, deviceDB) publicRoomsDB, err := storage.NewPublicRoomsServerDatabaseWithPubSub(string(base.Base.Cfg.Database.PublicRoomsAPI), base.LibP2PPubsub) if err != nil { diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index dd06cd3f9..4267cf166 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -32,7 +32,7 @@ func main() { deviceDB := base.CreateDeviceDB() keyDB := base.CreateKeyDB() federation := base.CreateFederationClient() - federationSender := base.CreateHTTPFederationSenderAPIs() + fsAPI := base.CreateHTTPFederationSenderAPIs() keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) alias, input, query := base.CreateHTTPRoomserverAPIs() @@ -42,7 +42,7 @@ func main() { federationapi.SetupFederationAPIComponent( base, accountDB, deviceDB, federation, &keyRing, - alias, input, query, asQuery, federationSender, eduProducer, + alias, input, query, asQuery, fsAPI, eduProducer, ) base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI)) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index f3e720a88..e806f6f3f 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -62,15 +62,16 @@ func main() { asQuery := appservice.SetupAppServiceAPIComponent( base, accountDB, deviceDB, federation, alias, query, transactions.New(), ) - fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query, input) + fsAPI := federationsender.SetupFederationSenderComponent(base, federation, query, input) + input.SetFederationSenderAPI(fsAPI) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, federation, &keyRing, alias, input, query, - eduInputAPI, asQuery, transactions.New(), fedSenderAPI, + eduInputAPI, asQuery, transactions.New(), fsAPI, ) eduProducer := producers.NewEDUServerProducer(eduInputAPI) - federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer) + federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fsAPI, eduProducer) mediaapi.SetupMediaAPIComponent(base, deviceDB) publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI)) if err != nil { diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go index 98410452f..3f9913e24 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-room-server/main.go @@ -30,7 +30,9 @@ func main() { federation := base.CreateFederationClient() keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - roomserver.SetupRoomServerComponent(base, keyRing, federation) + fsAPI := base.CreateHTTPFederationSenderAPIs() + _, input, _ := roomserver.SetupRoomServerComponent(base, keyRing, federation) + input.SetFederationSenderAPI(fsAPI) base.SetupAndServeHTTP(string(base.Cfg.Bind.RoomServer), string(base.Cfg.Listen.RoomServer)) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 6300d249f..7665138eb 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -129,6 +129,7 @@ func main() { base, accountDB, deviceDB, federation, alias, query, transactions.New(), ) fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query, input) + input.SetFederationSenderAPI(fedSenderAPI) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index 5e2d659bf..f245dd50b 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -149,12 +149,12 @@ func (b *BaseDendrite) CreateHTTPEDUServerAPIs() eduServerAPI.EDUServerInputAPI return e } -// CreateHTTPFederationSenderAPIs returns FederationSenderQueryAPI for hitting +// CreateHTTPFederationSenderAPIs returns FederationSenderInternalAPI for hitting // the federation sender over HTTP -func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.FederationSenderQueryAPI { - f, err := federationSenderAPI.NewFederationSenderQueryAPIHTTP(b.Cfg.FederationSenderURL(), b.httpClient) +func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.FederationSenderInternalAPI { + f, err := federationSenderAPI.NewFederationSenderInternalAPIHTTP(b.Cfg.FederationSenderURL(), b.httpClient) if err != nil { - logrus.WithError(err).Panic("NewFederationSenderQueryAPIHTTP failed", b.httpClient) + logrus.WithError(err).Panic("NewFederationSenderInternalAPIHTTP failed", b.httpClient) } return f } diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index ed96322b8..72e2b54ac 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -40,7 +40,7 @@ func SetupFederationAPIComponent( inputAPI roomserverAPI.RoomserverInputAPI, queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI, - federationSenderAPI federationSenderAPI.FederationSenderQueryAPI, + federationSenderAPI federationSenderAPI.FederationSenderInternalAPI, eduProducer *producers.EDUServerProducer, ) { roomserverProducer := producers.NewRoomserverProducer(inputAPI, queryAPI) diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go index 7cb50e525..13c92451d 100644 --- a/federationapi/routing/query.go +++ b/federationapi/routing/query.go @@ -33,7 +33,7 @@ func RoomAliasToID( federation *gomatrixserverlib.FederationClient, cfg *config.Dendrite, aliasAPI roomserverAPI.RoomserverAliasAPI, - senderAPI federationSenderAPI.FederationSenderQueryAPI, + senderAPI federationSenderAPI.FederationSenderInternalAPI, ) util.JSONResponse { roomAlias := httpReq.FormValue("room_alias") if roomAlias == "" { diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 83bac5550..ebaeec6e6 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -49,7 +49,7 @@ func Setup( asAPI appserviceAPI.AppServiceQueryAPI, producer *producers.RoomserverProducer, eduProducer *producers.EDUServerProducer, - federationSenderAPI federationSenderAPI.FederationSenderQueryAPI, + federationSenderAPI federationSenderAPI.FederationSenderInternalAPI, keys gomatrixserverlib.KeyRing, federation *gomatrixserverlib.FederationClient, accountDB accounts.Database, diff --git a/federationsender/api/api.go b/federationsender/api/api.go new file mode 100644 index 000000000..1340179e1 --- /dev/null +++ b/federationsender/api/api.go @@ -0,0 +1,53 @@ +package api + +import ( + "context" + "errors" + "net/http" +) + +// FederationSenderInternalAPI is used to query information from the federation sender. +type FederationSenderInternalAPI interface { + // Query the joined hosts and the membership events accounting for their participation in a room. + // Note that if a server has multiple users in the room, it will have multiple entries in the returned slice. + // See `QueryJoinedHostServerNamesInRoom` for a de-duplicated version. + QueryJoinedHostsInRoom( + ctx context.Context, + request *QueryJoinedHostsInRoomRequest, + response *QueryJoinedHostsInRoomResponse, + ) error + // Query the server names of the joined hosts in a room. + // Unlike QueryJoinedHostsInRoom, this function returns a de-duplicated slice + // containing only the server names (without information for membership events). + QueryJoinedHostServerNamesInRoom( + ctx context.Context, + request *QueryJoinedHostServerNamesInRoomRequest, + response *QueryJoinedHostServerNamesInRoomResponse, + ) error + // Handle an instruction to make_join & send_join with a remote server. + PerformJoinRequest( + ctx context.Context, + request *PerformJoinRequest, + response *PerformJoinResponse, + ) error + // Handle an instruction to make_leave & send_leave with a remote server. + PerformLeaveRequest( + ctx context.Context, + request *PerformLeaveRequest, + response *PerformLeaveResponse, + ) error +} + +// NewFederationSenderInternalAPIHTTP creates a FederationSenderInternalAPI implemented by talking to a HTTP POST API. +// If httpClient is nil an error is returned +func NewFederationSenderInternalAPIHTTP(federationSenderURL string, httpClient *http.Client) (FederationSenderInternalAPI, error) { + if httpClient == nil { + return nil, errors.New("NewFederationSenderInternalAPIHTTP: httpClient is ") + } + return &httpFederationSenderInternalAPI{federationSenderURL, httpClient}, nil +} + +type httpFederationSenderInternalAPI struct { + federationSenderURL string + httpClient *http.Client +} diff --git a/federationsender/api/perform.go b/federationsender/api/perform.go new file mode 100644 index 000000000..8c30ecbef --- /dev/null +++ b/federationsender/api/perform.go @@ -0,0 +1,56 @@ +package api + +import ( + "context" + + commonHTTP "github.com/matrix-org/dendrite/common/http" + "github.com/opentracing/opentracing-go" +) + +const ( + // FederationSenderPerformJoinRequestPath is the HTTP path for the PerformJoinRequest API. + FederationSenderPerformJoinRequestPath = "/api/federationsender/performJoinRequest" + + // FederationSenderPerformLeaveRequestPath is the HTTP path for the PerformLeaveRequest API. + FederationSenderPerformLeaveRequestPath = "/api/federationsender/performLeaveRequest" +) + +type PerformJoinRequest struct { + RoomID string `json:"room_id"` +} + +type PerformJoinResponse struct { +} + +// Handle an instruction to make_join & send_join with a remote server. +func (h *httpFederationSenderInternalAPI) PerformJoinRequest( + ctx context.Context, + request *PerformJoinRequest, + response *PerformJoinResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformJoinRequest") + defer span.Finish() + + apiURL := h.federationSenderURL + FederationSenderPerformJoinRequestPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +type PerformLeaveRequest struct { + RoomID string `json:"room_id"` +} + +type PerformLeaveResponse struct { +} + +// Handle an instruction to make_leave & send_leave with a remote server. +func (h *httpFederationSenderInternalAPI) PerformLeaveRequest( + ctx context.Context, + request *PerformLeaveRequest, + response *PerformLeaveResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformLeaveRequest") + defer span.Finish() + + apiURL := h.federationSenderURL + FederationSenderPerformLeaveRequestPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} diff --git a/federationsender/api/query.go b/federationsender/api/query.go index 7c0ca7ff2..7a58cc863 100644 --- a/federationsender/api/query.go +++ b/federationsender/api/query.go @@ -2,16 +2,20 @@ package api import ( "context" - "errors" - "net/http" commonHTTP "github.com/matrix-org/dendrite/common/http" + "github.com/matrix-org/dendrite/federationsender/types" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/dendrite/federationsender/types" "github.com/opentracing/opentracing-go" ) +// FederationSenderQueryJoinedHostsInRoomPath is the HTTP path for the QueryJoinedHostsInRoom API. +const FederationSenderQueryJoinedHostsInRoomPath = "/api/federationsender/queryJoinedHostsInRoom" + +// FederationSenderQueryJoinedHostServerNamesInRoomPath is the HTTP path for the QueryJoinedHostServerNamesInRoom API. +const FederationSenderQueryJoinedHostServerNamesInRoomPath = "/api/federationsender/queryJoinedHostServerNamesInRoom" + // QueryJoinedHostsInRoomRequest is a request to QueryJoinedHostsInRoom type QueryJoinedHostsInRoomRequest struct { RoomID string `json:"room_id"` @@ -22,6 +26,19 @@ type QueryJoinedHostsInRoomResponse struct { JoinedHosts []types.JoinedHost `json:"joined_hosts"` } +// QueryJoinedHostsInRoom implements FederationSenderInternalAPI +func (h *httpFederationSenderInternalAPI) QueryJoinedHostsInRoom( + ctx context.Context, + request *QueryJoinedHostsInRoomRequest, + response *QueryJoinedHostsInRoomResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostsInRoom") + defer span.Finish() + + apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostsInRoomPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + // QueryJoinedHostServerNamesRequest is a request to QueryJoinedHostServerNames type QueryJoinedHostServerNamesInRoomRequest struct { RoomID string `json:"room_id"` @@ -32,61 +49,8 @@ type QueryJoinedHostServerNamesInRoomResponse struct { ServerNames []gomatrixserverlib.ServerName `json:"server_names"` } -// FederationSenderQueryAPI is used to query information from the federation sender. -type FederationSenderQueryAPI interface { - // Query the joined hosts and the membership events accounting for their participation in a room. - // Note that if a server has multiple users in the room, it will have multiple entries in the returned slice. - // See `QueryJoinedHostServerNamesInRoom` for a de-duplicated version. - QueryJoinedHostsInRoom( - ctx context.Context, - request *QueryJoinedHostsInRoomRequest, - response *QueryJoinedHostsInRoomResponse, - ) error - // Query the server names of the joined hosts in a room. - // Unlike QueryJoinedHostsInRoom, this function returns a de-duplicated slice - // containing only the server names (without information for membership events). - QueryJoinedHostServerNamesInRoom( - ctx context.Context, - request *QueryJoinedHostServerNamesInRoomRequest, - response *QueryJoinedHostServerNamesInRoomResponse, - ) error -} - -// FederationSenderQueryJoinedHostsInRoomPath is the HTTP path for the QueryJoinedHostsInRoom API. -const FederationSenderQueryJoinedHostsInRoomPath = "/api/federationsender/queryJoinedHostsInRoom" - -// FederationSenderQueryJoinedHostServerNamesInRoomPath is the HTTP path for the QueryJoinedHostServerNamesInRoom API. -const FederationSenderQueryJoinedHostServerNamesInRoomPath = "/api/federationsender/queryJoinedHostServerNamesInRoom" - -// NewFederationSenderQueryAPIHTTP creates a FederationSenderQueryAPI implemented by talking to a HTTP POST API. -// If httpClient is nil an error is returned -func NewFederationSenderQueryAPIHTTP(federationSenderURL string, httpClient *http.Client) (FederationSenderQueryAPI, error) { - if httpClient == nil { - return nil, errors.New("NewFederationSenderQueryAPIHTTP: httpClient is ") - } - return &httpFederationSenderQueryAPI{federationSenderURL, httpClient}, nil -} - -type httpFederationSenderQueryAPI struct { - federationSenderURL string - httpClient *http.Client -} - -// QueryJoinedHostsInRoom implements FederationSenderQueryAPI -func (h *httpFederationSenderQueryAPI) QueryJoinedHostsInRoom( - ctx context.Context, - request *QueryJoinedHostsInRoomRequest, - response *QueryJoinedHostsInRoomResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostsInRoom") - defer span.Finish() - - apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostsInRoomPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// QueryJoinedHostServerNamesInRoom implements FederationSenderQueryAPI -func (h *httpFederationSenderQueryAPI) QueryJoinedHostServerNamesInRoom( +// QueryJoinedHostServerNamesInRoom implements FederationSenderInternalAPI +func (h *httpFederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom( ctx context.Context, request *QueryJoinedHostServerNamesInRoomRequest, response *QueryJoinedHostServerNamesInRoomResponse, diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index a06caf402..355775f8a 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -36,7 +36,7 @@ func SetupFederationSenderComponent( federation *gomatrixserverlib.FederationClient, rsQueryAPI roomserverAPI.RoomserverQueryAPI, rsInputAPI roomserverAPI.RoomserverInputAPI, -) api.FederationSenderQueryAPI { +) api.FederationSenderInternalAPI { federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender)) if err != nil { logrus.WithError(err).Panic("failed to connect to federation sender db") @@ -61,7 +61,7 @@ func SetupFederationSenderComponent( logrus.WithError(err).Panic("failed to start typing server consumer") } - queryAPI := query.FederationSenderQueryAPI{ + queryAPI := query.FederationSenderInternalAPI{ DB: federationSenderDB, } queryAPI.SetupHTTP(http.DefaultServeMux) diff --git a/federationsender/query/api.go b/federationsender/query/api.go new file mode 100644 index 000000000..e33bcc111 --- /dev/null +++ b/federationsender/query/api.go @@ -0,0 +1,77 @@ +package query + +import ( + "encoding/json" + "net/http" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/federationsender/storage" + rsAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/util" +) + +// FederationSenderInternalAPI is an implementation of api.FederationSenderInternalAPI +type FederationSenderInternalAPI struct { + api.FederationSenderInternalAPI + DB storage.Database + RoomserverInputAPI rsAPI.RoomserverInputAPI +} + +// SetupHTTP adds the FederationSenderInternalAPI handlers to the http.ServeMux. +func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) { + servMux.Handle( + api.FederationSenderQueryJoinedHostsInRoomPath, + common.MakeInternalAPI("QueryJoinedHostsInRoom", func(req *http.Request) util.JSONResponse { + var request api.QueryJoinedHostsInRoomRequest + var response api.QueryJoinedHostsInRoomResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := f.QueryJoinedHostsInRoom(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.FederationSenderQueryJoinedHostServerNamesInRoomPath, + common.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse { + var request api.QueryJoinedHostServerNamesInRoomRequest + var response api.QueryJoinedHostServerNamesInRoomResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := f.QueryJoinedHostServerNamesInRoom(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle(api.FederationSenderPerformJoinRequestPath, + common.MakeInternalAPI("PerformJoinRequest", func(req *http.Request) util.JSONResponse { + var request api.PerformJoinRequest + var response api.PerformJoinResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := f.PerformJoinRequest(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle(api.FederationSenderPerformLeaveRequestPath, + common.MakeInternalAPI("PerformLeaveRequest", func(req *http.Request) util.JSONResponse { + var request api.PerformLeaveRequest + var response api.PerformLeaveResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := f.PerformLeaveRequest(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) +} diff --git a/federationsender/query/perform.go b/federationsender/query/perform.go new file mode 100644 index 000000000..2486873c2 --- /dev/null +++ b/federationsender/query/perform.go @@ -0,0 +1,25 @@ +package query + +import ( + "context" + + "github.com/matrix-org/dendrite/federationsender/api" +) + +// PerformJoinRequest implements api.FederationSenderInternalAPI +func (r *FederationSenderInternalAPI) PerformJoinRequest( + ctx context.Context, + request *api.PerformJoinRequest, + response *api.PerformJoinResponse, +) (err error) { + return nil +} + +// PerformLeaveRequest implements api.FederationSenderInternalAPI +func (r *FederationSenderInternalAPI) PerformLeaveRequest( + ctx context.Context, + request *api.PerformLeaveRequest, + response *api.PerformLeaveResponse, +) (err error) { + return nil +} diff --git a/federationsender/query/query.go b/federationsender/query/query.go index 8c35bb29e..ec668204d 100644 --- a/federationsender/query/query.go +++ b/federationsender/query/query.go @@ -2,30 +2,13 @@ package query import ( "context" - "encoding/json" - "net/http" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/federationsender/api" - "github.com/matrix-org/dendrite/federationsender/types" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" ) -// FederationSenderQueryDatabase has the APIs needed to implement the query API. -type FederationSenderQueryDatabase interface { - GetJoinedHosts( - ctx context.Context, roomID string, - ) ([]types.JoinedHost, error) -} - -// FederationSenderQueryAPI is an implementation of api.FederationSenderQueryAPI -type FederationSenderQueryAPI struct { - DB FederationSenderQueryDatabase -} - -// QueryJoinedHostsInRoom implements api.FederationSenderQueryAPI -func (f *FederationSenderQueryAPI) QueryJoinedHostsInRoom( +// QueryJoinedHostsInRoom implements api.FederationSenderInternalAPI +func (f *FederationSenderInternalAPI) QueryJoinedHostsInRoom( ctx context.Context, request *api.QueryJoinedHostsInRoomRequest, response *api.QueryJoinedHostsInRoomResponse, @@ -34,8 +17,8 @@ func (f *FederationSenderQueryAPI) QueryJoinedHostsInRoom( return } -// QueryJoinedHostServerNamesInRoom implements api.FederationSenderQueryAPI -func (f *FederationSenderQueryAPI) QueryJoinedHostServerNamesInRoom( +// QueryJoinedHostServerNamesInRoom implements api.FederationSenderInternalAPI +func (f *FederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom( ctx context.Context, request *api.QueryJoinedHostServerNamesInRoomRequest, response *api.QueryJoinedHostServerNamesInRoomResponse, @@ -54,35 +37,3 @@ func (f *FederationSenderQueryAPI) QueryJoinedHostServerNamesInRoom( return } - -// SetupHTTP adds the FederationSenderQueryAPI handlers to the http.ServeMux. -func (f *FederationSenderQueryAPI) SetupHTTP(servMux *http.ServeMux) { - servMux.Handle( - api.FederationSenderQueryJoinedHostsInRoomPath, - common.MakeInternalAPI("QueryJoinedHostsInRoom", func(req *http.Request) util.JSONResponse { - var request api.QueryJoinedHostsInRoomRequest - var response api.QueryJoinedHostsInRoomResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := f.QueryJoinedHostsInRoom(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.FederationSenderQueryJoinedHostServerNamesInRoomPath, - common.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse { - var request api.QueryJoinedHostServerNamesInRoomRequest - var response api.QueryJoinedHostServerNamesInRoomResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := f.QueryJoinedHostServerNamesInRoom(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) -} diff --git a/roomserver/api/input.go b/roomserver/api/input.go index bb4e040de..d9cffad27 100644 --- a/roomserver/api/input.go +++ b/roomserver/api/input.go @@ -21,6 +21,7 @@ import ( "net/http" commonHTTP "github.com/matrix-org/dendrite/common/http" + fsAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/gomatrixserverlib" opentracing "github.com/opentracing/opentracing-go" ) @@ -106,6 +107,9 @@ type InputRoomEventsResponse struct { // RoomserverInputAPI is used to write events to the room server. type RoomserverInputAPI interface { + // needed to avoid chicken and egg scenario when setting up the + // interdependencies between the roomserver and the FS input API + SetFederationSenderAPI(fsInputAPI fsAPI.FederationSenderInternalAPI) InputRoomEvents( ctx context.Context, request *InputRoomEventsRequest, @@ -122,12 +126,22 @@ func NewRoomserverInputAPIHTTP(roomserverURL string, httpClient *http.Client) (R if httpClient == nil { return nil, errors.New("NewRoomserverInputAPIHTTP: httpClient is ") } - return &httpRoomserverInputAPI{roomserverURL, httpClient}, nil + return &httpRoomserverInputAPI{roomserverURL, httpClient, nil}, nil } type httpRoomserverInputAPI struct { roomserverURL string httpClient *http.Client + // The federation sender API allows us to send federation + // requests from the new perform input requests, still TODO. + fsInputAPI fsAPI.FederationSenderInternalAPI +} + +// SetFederationSenderInputAPI passes in a federation sender input API reference +// so that we can avoid the chicken-and-egg problem of both the roomserver input API +// and the federation sender input API being interdependent. +func (h *httpRoomserverInputAPI) SetFederationSenderAPI(fsInputAPI fsAPI.FederationSenderInternalAPI) { + h.fsInputAPI = fsInputAPI } // InputRoomEvents implements RoomserverInputAPI diff --git a/roomserver/input/input.go b/roomserver/input/input.go index cb588380a..20b6afc4b 100644 --- a/roomserver/input/input.go +++ b/roomserver/input/input.go @@ -26,6 +26,8 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/util" + + fsAPI "github.com/matrix-org/dendrite/federationsender/api" ) // RoomserverInputAPI implements api.RoomserverInputAPI @@ -37,6 +39,16 @@ type RoomserverInputAPI struct { OutputRoomEventTopic string // Protects calls to processRoomEvent mutex sync.Mutex + // The federation sender API allows us to send federation + // requests from the new perform input requests, still TODO. + fsAPI fsAPI.FederationSenderInternalAPI +} + +// SetFederationSenderInputAPI passes in a federation sender input API reference +// so that we can avoid the chicken-and-egg problem of both the roomserver input API +// and the federation sender input API being interdependent. +func (r *RoomserverInputAPI) SetFederationSenderAPI(fsAPI fsAPI.FederationSenderInternalAPI) { + r.fsAPI = fsAPI } // WriteOutputEvents implements OutputRoomEventWriter diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index f6a373a4f..6fb2caff9 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -21,6 +21,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" asQuery "github.com/matrix-org/dendrite/appservice/query" + "github.com/matrix-org/dendrite/common/basecomponent" "github.com/matrix-org/dendrite/roomserver/alias" "github.com/matrix-org/dendrite/roomserver/input" From 64e94e9a6f0a138e7fe771f540b57988bc344b59 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 29 Apr 2020 15:29:39 +0100 Subject: [PATCH 05/12] Join room support in federation sender (#989) * Implement PerformJoinRequest * Rename perform functions * Check send join response * Temporary wiring to test federation sender room joins * Actually pass through the config * Make sure membership content shows join --- clientapi/routing/joinroom.go | 124 ++---------------- clientapi/routing/routing.go | 3 +- cmd/dendrite-demo-libp2p/main.go | 2 +- cmd/dendrite-federation-sender-server/main.go | 5 +- cmd/dendrite-monolith-server/main.go | 2 +- cmd/dendritejs/main.go | 2 +- federationsender/api/api.go | 4 +- federationsender/api/perform.go | 10 +- federationsender/federationsender.go | 9 +- federationsender/producers/roomserver.go | 36 +++++ federationsender/query/api.go | 30 ++++- federationsender/query/perform.go | 99 +++++++++++++- federationsender/query/perform/join.go | 70 ++++++++++ federationsender/query/query.go | 4 +- 14 files changed, 265 insertions(+), 135 deletions(-) create mode 100644 federationsender/query/perform/join.go diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index d0dee7c2a..f55d1b6ab 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -27,12 +27,11 @@ import ( "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" - "github.com/matrix-org/dendrite/roomserver/api" + federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" - "github.com/sirupsen/logrus" ) // JoinRoomByIDOrAlias implements the "/join/{roomIDOrAlias}" API. @@ -46,6 +45,7 @@ func JoinRoomByIDOrAlias( producer *producers.RoomserverProducer, queryAPI roomserverAPI.RoomserverQueryAPI, aliasAPI roomserverAPI.RoomserverAliasAPI, + fsAPI federationSenderAPI.FederationSenderInternalAPI, keyRing gomatrixserverlib.KeyRing, accountDB accounts.Database, ) util.JSONResponse { @@ -79,7 +79,8 @@ func JoinRoomByIDOrAlias( content["avatar_url"] = profile.AvatarURL r := joinRoomReq{ - req, evTime, content, device.UserID, cfg, federation, producer, queryAPI, aliasAPI, keyRing, + req, evTime, content, device.UserID, cfg, federation, producer, + queryAPI, aliasAPI, fsAPI, keyRing, } if strings.HasPrefix(roomIDOrAlias, "!") { @@ -107,6 +108,7 @@ type joinRoomReq struct { producer *producers.RoomserverProducer queryAPI roomserverAPI.RoomserverQueryAPI aliasAPI roomserverAPI.RoomserverAliasAPI + fsAPI federationSenderAPI.FederationSenderInternalAPI keyRing gomatrixserverlib.KeyRing } @@ -326,71 +328,15 @@ func (r joinRoomReq) joinRoomUsingServers( // server was invalid this returns an error. // Otherwise this returns a JSONResponse. func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib.ServerName) (*util.JSONResponse, error) { - // Ask the room server for information about room versions. - var request api.QueryRoomVersionCapabilitiesRequest - var response api.QueryRoomVersionCapabilitiesResponse - if err := r.queryAPI.QueryRoomVersionCapabilities(r.req.Context(), &request, &response); err != nil { + fedJoinReq := federationSenderAPI.PerformJoinRequest{ + RoomID: roomID, + UserID: r.userID, + ServerName: server, + } + fedJoinRes := federationSenderAPI.PerformJoinResponse{} + if err := r.fsAPI.PerformJoin(r.req.Context(), &fedJoinReq, &fedJoinRes); err != nil { return nil, err } - var supportedVersions []gomatrixserverlib.RoomVersion - for version := range response.AvailableRoomVersions { - supportedVersions = append(supportedVersions, version) - } - respMakeJoin, err := r.federation.MakeJoin(r.req.Context(), server, roomID, r.userID, supportedVersions) - if err != nil { - // TODO: Check if the user was not allowed to join the room. - return nil, fmt.Errorf("r.federation.MakeJoin: %w", err) - } - - // Set all the fields to be what they should be, this should be a no-op - // but it's possible that the remote server returned us something "odd" - err = r.writeToBuilder(&respMakeJoin.JoinEvent, roomID) - if err != nil { - return nil, fmt.Errorf("r.writeToBuilder: %w", err) - } - - if respMakeJoin.RoomVersion == "" { - respMakeJoin.RoomVersion = gomatrixserverlib.RoomVersionV1 - } - if _, err = respMakeJoin.RoomVersion.EventFormat(); err != nil { - return &util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.UnsupportedRoomVersion( - fmt.Sprintf("Room version '%s' is not supported", respMakeJoin.RoomVersion), - ), - }, nil - } - - event, err := respMakeJoin.JoinEvent.Build( - r.evTime, r.cfg.Matrix.ServerName, r.cfg.Matrix.KeyID, - r.cfg.Matrix.PrivateKey, respMakeJoin.RoomVersion, - ) - if err != nil { - return nil, fmt.Errorf("respMakeJoin.JoinEvent.Build: %w", err) - } - - respSendJoin, err := r.federation.SendJoin(r.req.Context(), server, event, respMakeJoin.RoomVersion) - if err != nil { - return nil, fmt.Errorf("r.federation.SendJoin: %w", err) - } - - if err = r.checkSendJoinResponse(event, server, respMakeJoin, respSendJoin); err != nil { - return nil, err - } - - util.GetLogger(r.req.Context()).WithFields(logrus.Fields{ - "room_id": roomID, - "num_auth_events": len(respSendJoin.AuthEvents), - "num_state_events": len(respSendJoin.StateEvents), - }).Info("Room join signature and auth verification passed") - - if err = r.producer.SendEventWithState( - r.req.Context(), - respSendJoin.ToRespState(), - event.Headered(respMakeJoin.RoomVersion), - ); err != nil { - util.GetLogger(r.req.Context()).WithError(err).Error("r.producer.SendEventWithState") - } return &util.JSONResponse{ Code: http.StatusOK, @@ -400,49 +346,3 @@ func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib }{roomID}, }, nil } - -// checkSendJoinResponse checks that all of the signatures are correct -// and that the join is allowed by the supplied state. -func (r joinRoomReq) checkSendJoinResponse( - event gomatrixserverlib.Event, - server gomatrixserverlib.ServerName, - respMakeJoin gomatrixserverlib.RespMakeJoin, - respSendJoin gomatrixserverlib.RespSendJoin, -) error { - // A list of events that we have retried, if they were not included in - // the auth events supplied in the send_join. - retries := map[string]bool{} - -retryCheck: - // TODO: Can we expand Check here to return a list of missing auth - // events rather than failing one at a time? - if err := respSendJoin.Check(r.req.Context(), r.keyRing, event); err != nil { - switch e := err.(type) { - case gomatrixserverlib.MissingAuthEventError: - // Check that we haven't already retried for this event, prevents - // us from ending up in endless loops - if !retries[e.AuthEventID] { - // Ask the server that we're talking to right now for the event - tx, txerr := r.federation.GetEvent(r.req.Context(), server, e.AuthEventID) - if txerr != nil { - return fmt.Errorf("r.federation.GetEvent: %w", txerr) - } - // For each event returned, add it to the auth events. - for _, pdu := range tx.PDUs { - ev, everr := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, respMakeJoin.RoomVersion) - if everr != nil { - return fmt.Errorf("gomatrixserverlib.NewEventFromUntrustedJSON: %w", everr) - } - respSendJoin.AuthEvents = append(respSendJoin.AuthEvents, ev) - } - // Mark the event as retried and then give the check another go. - retries[e.AuthEventID] = true - goto retryCheck - } - return fmt.Errorf("respSendJoin (after retries): %w", e) - default: - return fmt.Errorf("respSendJoin: %w", err) - } - } - return nil -} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 9ab22cbec..e62b51935 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -101,7 +101,8 @@ func Setup( return util.ErrorResponse(err) } return JoinRoomByIDOrAlias( - req, device, vars["roomIDOrAlias"], cfg, federation, producer, queryAPI, aliasAPI, keyRing, accountDB, + req, device, vars["roomIDOrAlias"], cfg, federation, producer, + queryAPI, aliasAPI, federationSender, keyRing, accountDB, ) }), ).Methods(http.MethodPost, http.MethodOptions) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index b9fbfc53e..a2a4675b3 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -153,7 +153,7 @@ func main() { asQuery := appservice.SetupAppServiceAPIComponent( &base.Base, accountDB, deviceDB, federation, alias, query, transactions.New(), ) - fsAPI := federationsender.SetupFederationSenderComponent(&base.Base, federation, query, input) + fsAPI := federationsender.SetupFederationSenderComponent(&base.Base, federation, query, input, &keyRing) clientapi.SetupClientAPIComponent( &base.Base, deviceDB, accountDB, diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-federation-sender-server/main.go index 1593afaa5..f8d43b990 100644 --- a/cmd/dendrite-federation-sender-server/main.go +++ b/cmd/dendrite-federation-sender-server/main.go @@ -16,6 +16,7 @@ package main import ( "github.com/matrix-org/dendrite/common/basecomponent" + "github.com/matrix-org/dendrite/common/keydb" "github.com/matrix-org/dendrite/federationsender" ) @@ -25,11 +26,13 @@ func main() { defer base.Close() // nolint: errcheck federation := base.CreateFederationClient() + keyDB := base.CreateKeyDB() + keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) _, input, query := base.CreateHTTPRoomserverAPIs() federationsender.SetupFederationSenderComponent( - base, federation, query, input, + base, federation, query, input, &keyRing, ) base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationSender), string(base.Cfg.Listen.FederationSender)) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index e806f6f3f..f43f8b04c 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -62,7 +62,7 @@ func main() { asQuery := appservice.SetupAppServiceAPIComponent( base, accountDB, deviceDB, federation, alias, query, transactions.New(), ) - fsAPI := federationsender.SetupFederationSenderComponent(base, federation, query, input) + fsAPI := federationsender.SetupFederationSenderComponent(base, federation, query, input, &keyRing) input.SetFederationSenderAPI(fsAPI) clientapi.SetupClientAPIComponent( diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 7665138eb..1f2f20fb4 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -128,7 +128,7 @@ func main() { asQuery := appservice.SetupAppServiceAPIComponent( base, accountDB, deviceDB, federation, alias, query, transactions.New(), ) - fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query, input) + fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query, input, &keyRing) input.SetFederationSenderAPI(fedSenderAPI) clientapi.SetupClientAPIComponent( diff --git a/federationsender/api/api.go b/federationsender/api/api.go index 1340179e1..10dc66da2 100644 --- a/federationsender/api/api.go +++ b/federationsender/api/api.go @@ -25,13 +25,13 @@ type FederationSenderInternalAPI interface { response *QueryJoinedHostServerNamesInRoomResponse, ) error // Handle an instruction to make_join & send_join with a remote server. - PerformJoinRequest( + PerformJoin( ctx context.Context, request *PerformJoinRequest, response *PerformJoinResponse, ) error // Handle an instruction to make_leave & send_leave with a remote server. - PerformLeaveRequest( + PerformLeave( ctx context.Context, request *PerformLeaveRequest, response *PerformLeaveResponse, diff --git a/federationsender/api/perform.go b/federationsender/api/perform.go index 8c30ecbef..87736f294 100644 --- a/federationsender/api/perform.go +++ b/federationsender/api/perform.go @@ -4,6 +4,7 @@ import ( "context" commonHTTP "github.com/matrix-org/dendrite/common/http" + "github.com/matrix-org/gomatrixserverlib" "github.com/opentracing/opentracing-go" ) @@ -16,14 +17,17 @@ const ( ) type PerformJoinRequest struct { - RoomID string `json:"room_id"` + RoomID string `json:"room_id"` + UserID string `json:"user_id"` + ServerName gomatrixserverlib.ServerName `json:"server_name"` + Content map[string]interface{} `json:"content"` } type PerformJoinResponse struct { } // Handle an instruction to make_join & send_join with a remote server. -func (h *httpFederationSenderInternalAPI) PerformJoinRequest( +func (h *httpFederationSenderInternalAPI) PerformJoin( ctx context.Context, request *PerformJoinRequest, response *PerformJoinResponse, @@ -43,7 +47,7 @@ type PerformLeaveResponse struct { } // Handle an instruction to make_leave & send_leave with a remote server. -func (h *httpFederationSenderInternalAPI) PerformLeaveRequest( +func (h *httpFederationSenderInternalAPI) PerformLeave( ctx context.Context, request *PerformLeaveRequest, response *PerformLeaveResponse, diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index 355775f8a..aa9a7bc9c 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -36,6 +36,7 @@ func SetupFederationSenderComponent( federation *gomatrixserverlib.FederationClient, rsQueryAPI roomserverAPI.RoomserverQueryAPI, rsInputAPI roomserverAPI.RoomserverInputAPI, + keyRing *gomatrixserverlib.KeyRing, ) api.FederationSenderInternalAPI { federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender)) if err != nil { @@ -61,10 +62,10 @@ func SetupFederationSenderComponent( logrus.WithError(err).Panic("failed to start typing server consumer") } - queryAPI := query.FederationSenderInternalAPI{ - DB: federationSenderDB, - } + queryAPI := query.NewFederationSenderInternalAPI( + federationSenderDB, base.Cfg, roomserverProducer, federation, keyRing, + ) queryAPI.SetupHTTP(http.DefaultServeMux) - return &queryAPI + return queryAPI } diff --git a/federationsender/producers/roomserver.go b/federationsender/producers/roomserver.go index 0395f9628..ff4cda5b5 100644 --- a/federationsender/producers/roomserver.go +++ b/federationsender/producers/roomserver.go @@ -54,6 +54,42 @@ func (c *RoomserverProducer) SendInviteResponse( return c.SendInputRoomEvents(ctx, []api.InputRoomEvent{ire}) } +// SendEventWithState writes an event with KindNew to the roomserver input log +// with the state at the event as KindOutlier before it. +func (c *RoomserverProducer) SendEventWithState( + ctx context.Context, state gomatrixserverlib.RespState, event gomatrixserverlib.HeaderedEvent, +) error { + outliers, err := state.Events() + if err != nil { + return err + } + + var ires []api.InputRoomEvent + for _, outlier := range outliers { + ires = append(ires, api.InputRoomEvent{ + Kind: api.KindOutlier, + Event: outlier.Headered(event.RoomVersion), + AuthEventIDs: outlier.AuthEventIDs(), + }) + } + + stateEventIDs := make([]string, len(state.StateEvents)) + for i := range state.StateEvents { + stateEventIDs[i] = state.StateEvents[i].EventID() + } + + ires = append(ires, api.InputRoomEvent{ + Kind: api.KindNew, + Event: event, + AuthEventIDs: event.AuthEventIDs(), + HasState: true, + StateEventIDs: stateEventIDs, + }) + + _, err = c.SendInputRoomEvents(ctx, ires) + return err +} + // SendInputRoomEvents writes the given input room events to the roomserver input API. func (c *RoomserverProducer) SendInputRoomEvents( ctx context.Context, ires []api.InputRoomEvent, diff --git a/federationsender/query/api.go b/federationsender/query/api.go index e33bcc111..e92453f98 100644 --- a/federationsender/query/api.go +++ b/federationsender/query/api.go @@ -5,17 +5,37 @@ import ( "net/http" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/federationsender/producers" "github.com/matrix-org/dendrite/federationsender/storage" - rsAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) // FederationSenderInternalAPI is an implementation of api.FederationSenderInternalAPI type FederationSenderInternalAPI struct { api.FederationSenderInternalAPI - DB storage.Database - RoomserverInputAPI rsAPI.RoomserverInputAPI + db storage.Database + cfg *config.Dendrite + producer *producers.RoomserverProducer + federation *gomatrixserverlib.FederationClient + keyRing *gomatrixserverlib.KeyRing +} + +func NewFederationSenderInternalAPI( + db storage.Database, cfg *config.Dendrite, + producer *producers.RoomserverProducer, + federation *gomatrixserverlib.FederationClient, + keyRing *gomatrixserverlib.KeyRing, +) *FederationSenderInternalAPI { + return &FederationSenderInternalAPI{ + db: db, + cfg: cfg, + producer: producer, + federation: federation, + keyRing: keyRing, + } } // SetupHTTP adds the FederationSenderInternalAPI handlers to the http.ServeMux. @@ -55,7 +75,7 @@ func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) { if err := json.NewDecoder(req.Body).Decode(&request); err != nil { return util.MessageResponse(http.StatusBadRequest, err.Error()) } - if err := f.PerformJoinRequest(req.Context(), &request, &response); err != nil { + if err := f.PerformJoin(req.Context(), &request, &response); err != nil { return util.ErrorResponse(err) } return util.JSONResponse{Code: http.StatusOK, JSON: &response} @@ -68,7 +88,7 @@ func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) { if err := json.NewDecoder(req.Body).Decode(&request); err != nil { return util.MessageResponse(http.StatusBadRequest, err.Error()) } - if err := f.PerformLeaveRequest(req.Context(), &request, &response); err != nil { + if err := f.PerformLeave(req.Context(), &request, &response); err != nil { return util.ErrorResponse(err) } return util.JSONResponse{Code: http.StatusOK, JSON: &response} diff --git a/federationsender/query/perform.go b/federationsender/query/perform.go index 2486873c2..d39fef5ed 100644 --- a/federationsender/query/perform.go +++ b/federationsender/query/perform.go @@ -2,21 +2,116 @@ package query import ( "context" + "fmt" + "time" "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/federationsender/query/perform" + "github.com/matrix-org/dendrite/roomserver/version" + "github.com/matrix-org/gomatrixserverlib" ) // PerformJoinRequest implements api.FederationSenderInternalAPI -func (r *FederationSenderInternalAPI) PerformJoinRequest( +func (r *FederationSenderInternalAPI) PerformJoin( ctx context.Context, request *api.PerformJoinRequest, response *api.PerformJoinResponse, ) (err error) { + // Look up the supported room versions. + var supportedVersions []gomatrixserverlib.RoomVersion + for version := range version.SupportedRoomVersions() { + supportedVersions = append(supportedVersions, version) + } + + // Try to perform a make_join using the information supplied in the + // request. + respMakeJoin, err := r.federation.MakeJoin( + ctx, + request.ServerName, + request.RoomID, + request.UserID, + supportedVersions, + ) + if err != nil { + // TODO: Check if the user was not allowed to join the room. + return fmt.Errorf("r.federation.MakeJoin: %w", err) + } + + // Set all the fields to be what they should be, this should be a no-op + // but it's possible that the remote server returned us something "odd" + respMakeJoin.JoinEvent.Type = "m.room.member" + respMakeJoin.JoinEvent.Sender = request.UserID + respMakeJoin.JoinEvent.StateKey = &request.UserID + respMakeJoin.JoinEvent.RoomID = request.RoomID + respMakeJoin.JoinEvent.Redacts = "" + if request.Content == nil { + request.Content = map[string]interface{}{} + } + request.Content["membership"] = "join" + if err = respMakeJoin.JoinEvent.SetContent(request.Content); err != nil { + return fmt.Errorf("respMakeJoin.JoinEvent.SetContent: %w", err) + } + if err = respMakeJoin.JoinEvent.SetUnsigned(struct{}{}); err != nil { + return fmt.Errorf("respMakeJoin.JoinEvent.SetUnsigned: %w", err) + } + + // Work out if we support the room version that has been supplied in + // the make_join response. + if respMakeJoin.RoomVersion == "" { + respMakeJoin.RoomVersion = gomatrixserverlib.RoomVersionV1 + } + if _, err = respMakeJoin.RoomVersion.EventFormat(); err != nil { + return fmt.Errorf("respMakeJoin.RoomVersion.EventFormat: %w", err) + } + + // Build the join event. + event, err := respMakeJoin.JoinEvent.Build( + time.Now(), + r.cfg.Matrix.ServerName, + r.cfg.Matrix.KeyID, + r.cfg.Matrix.PrivateKey, + respMakeJoin.RoomVersion, + ) + if err != nil { + return fmt.Errorf("respMakeJoin.JoinEvent.Build: %w", err) + } + + // Try to perform a send_join using the newly built event. + respSendJoin, err := r.federation.SendJoin( + ctx, + request.ServerName, + event, + respMakeJoin.RoomVersion, + ) + if err != nil { + return fmt.Errorf("r.federation.SendJoin: %w", err) + } + + // Check that the send_join response was valid. + joinCtx := perform.JoinContext(r.federation, r.keyRing) + if err = joinCtx.CheckSendJoinResponse( + ctx, event, request.ServerName, respMakeJoin, respSendJoin, + ); err != nil { + return fmt.Errorf("perform.JoinRequest.CheckSendJoinResponse: %w", err) + } + + // If we successfully performed a send_join above then the other + // server now thinks we're a part of the room. Send the newly + // returned state to the roomserver to update our local view. + if err = r.producer.SendEventWithState( + ctx, + respSendJoin.ToRespState(), + event.Headered(respMakeJoin.RoomVersion), + ); err != nil { + return fmt.Errorf("r.producer.SendEventWithState: %w", err) + } + + // Everything went to plan. return nil } // PerformLeaveRequest implements api.FederationSenderInternalAPI -func (r *FederationSenderInternalAPI) PerformLeaveRequest( +func (r *FederationSenderInternalAPI) PerformLeave( ctx context.Context, request *api.PerformLeaveRequest, response *api.PerformLeaveResponse, diff --git a/federationsender/query/perform/join.go b/federationsender/query/perform/join.go new file mode 100644 index 000000000..3c7ef0765 --- /dev/null +++ b/federationsender/query/perform/join.go @@ -0,0 +1,70 @@ +package perform + +import ( + "context" + "fmt" + + "github.com/matrix-org/gomatrixserverlib" +) + +// This file contains helpers for the PerformJoin function. + +type joinContext struct { + federation *gomatrixserverlib.FederationClient + keyRing *gomatrixserverlib.KeyRing +} + +// Returns a new join context. +func JoinContext(f *gomatrixserverlib.FederationClient, k *gomatrixserverlib.KeyRing) *joinContext { + return &joinContext{ + federation: f, + keyRing: k, + } +} + +// checkSendJoinResponse checks that all of the signatures are correct +// and that the join is allowed by the supplied state. +func (r joinContext) CheckSendJoinResponse( + ctx context.Context, + event gomatrixserverlib.Event, + server gomatrixserverlib.ServerName, + respMakeJoin gomatrixserverlib.RespMakeJoin, + respSendJoin gomatrixserverlib.RespSendJoin, +) error { + // A list of events that we have retried, if they were not included in + // the auth events supplied in the send_join. + retries := map[string]bool{} + +retryCheck: + // TODO: Can we expand Check here to return a list of missing auth + // events rather than failing one at a time? + if err := respSendJoin.Check(ctx, r.keyRing, event); err != nil { + switch e := err.(type) { + case gomatrixserverlib.MissingAuthEventError: + // Check that we haven't already retried for this event, prevents + // us from ending up in endless loops + if !retries[e.AuthEventID] { + // Ask the server that we're talking to right now for the event + tx, txerr := r.federation.GetEvent(ctx, server, e.AuthEventID) + if txerr != nil { + return fmt.Errorf("r.federation.GetEvent: %w", txerr) + } + // For each event returned, add it to the auth events. + for _, pdu := range tx.PDUs { + ev, everr := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, respMakeJoin.RoomVersion) + if everr != nil { + return fmt.Errorf("gomatrixserverlib.NewEventFromUntrustedJSON: %w", everr) + } + respSendJoin.AuthEvents = append(respSendJoin.AuthEvents, ev) + } + // Mark the event as retried and then give the check another go. + retries[e.AuthEventID] = true + goto retryCheck + } + return fmt.Errorf("respSendJoin (after retries): %w", e) + default: + return fmt.Errorf("respSendJoin: %w", err) + } + } + return nil +} diff --git a/federationsender/query/query.go b/federationsender/query/query.go index ec668204d..004ad156d 100644 --- a/federationsender/query/query.go +++ b/federationsender/query/query.go @@ -13,7 +13,7 @@ func (f *FederationSenderInternalAPI) QueryJoinedHostsInRoom( request *api.QueryJoinedHostsInRoomRequest, response *api.QueryJoinedHostsInRoomResponse, ) (err error) { - response.JoinedHosts, err = f.DB.GetJoinedHosts(ctx, request.RoomID) + response.JoinedHosts, err = f.db.GetJoinedHosts(ctx, request.RoomID) return } @@ -23,7 +23,7 @@ func (f *FederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom( request *api.QueryJoinedHostServerNamesInRoomRequest, response *api.QueryJoinedHostServerNamesInRoomResponse, ) (err error) { - joinedHosts, err := f.DB.GetJoinedHosts(ctx, request.RoomID) + joinedHosts, err := f.db.GetJoinedHosts(ctx, request.RoomID) if err != nil { return } From 458b3647815f0f2c6930611961431a9fb4390fba Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 29 Apr 2020 15:33:17 +0100 Subject: [PATCH 06/12] Update gomatrixserverlib --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 56c58c6c8..5eb54ceb5 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200428112024-9f47f9bfa4b2 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200429143250-5df6426424bd github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index de844b2a1..1511b927f 100644 --- a/go.sum +++ b/go.sum @@ -367,8 +367,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200428112024-9f47f9bfa4b2 h1:sy2QOqJhb4WXzq8bJhsCntAUYb64Dl6txsFtXWtxxSg= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200428112024-9f47f9bfa4b2/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200429143250-5df6426424bd h1:YA+1/Y/NK6dHAxRamybQiE6HToTC+5ddPCO4UI7pmH0= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200429143250-5df6426424bd/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= From 4ad52c67cacc21997f49decd57e3105beb8ab62d Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 29 Apr 2020 18:41:45 +0100 Subject: [PATCH 07/12] Honour history_visibility when backfilling (#990) * Make backfill work for shared history visibility * fetch missing state on backfill to remember snapshots correctly * Fix gmsl to not mux in auth events into room state * Whoops * Linting --- federationapi/routing/events.go | 11 ++- federationapi/routing/routing.go | 2 +- go.mod | 2 +- go.sum | 4 +- roomserver/auth/auth.go | 4 +- roomserver/query/backfill.go | 81 ++++++++++++++-- roomserver/query/query.go | 108 +++++++++++++++++---- roomserver/state/state.go | 5 +- roomserver/storage/sqlite3/events_table.go | 28 +++--- roomserver/storage/sqlite3/storage.go | 2 +- 10 files changed, 195 insertions(+), 52 deletions(-) diff --git a/federationapi/routing/events.go b/federationapi/routing/events.go index a91528b3d..03492db45 100644 --- a/federationapi/routing/events.go +++ b/federationapi/routing/events.go @@ -16,7 +16,9 @@ package routing import ( "context" + "encoding/json" "net/http" + "time" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -29,13 +31,20 @@ func GetEvent( request *gomatrixserverlib.FederationRequest, query api.RoomserverQueryAPI, eventID string, + origin gomatrixserverlib.ServerName, ) util.JSONResponse { event, err := getEvent(ctx, request, query, eventID) if err != nil { return *err } - return util.JSONResponse{Code: http.StatusOK, JSON: event} + return util.JSONResponse{Code: http.StatusOK, JSON: gomatrixserverlib.Transaction{ + Origin: origin, + OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()), + PDUs: []json.RawMessage{ + event.JSON(), + }, + }} } // getEvent returns the requested event, diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index ebaeec6e6..e0f842f18 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -126,7 +126,7 @@ func Setup( return util.ErrorResponse(err) } return GetEvent( - httpReq.Context(), request, query, vars["eventID"], + httpReq.Context(), request, query, vars["eventID"], cfg.Matrix.ServerName, ) }, )).Methods(http.MethodGet) diff --git a/go.mod b/go.mod index 5eb54ceb5..d20089e8a 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200429143250-5df6426424bd + github.com/matrix-org/gomatrixserverlib v0.0.0-20200429162354-392f0b1b7421 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index 1511b927f..da0e02d6f 100644 --- a/go.sum +++ b/go.sum @@ -367,8 +367,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200429143250-5df6426424bd h1:YA+1/Y/NK6dHAxRamybQiE6HToTC+5ddPCO4UI7pmH0= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200429143250-5df6426424bd/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200429162354-392f0b1b7421 h1:4zP29YlpfEtJ9a7sZ33Mf0FJInD2N3/KzDcLa62bRKc= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200429162354-392f0b1b7421/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= diff --git a/roomserver/auth/auth.go b/roomserver/auth/auth.go index 615a94b3c..fdcf9f062 100644 --- a/roomserver/auth/auth.go +++ b/roomserver/auth/auth.go @@ -27,7 +27,7 @@ func IsServerAllowed( serverCurrentlyInRoom bool, authEvents []gomatrixserverlib.Event, ) bool { - historyVisibility := historyVisibilityForRoom(authEvents) + historyVisibility := HistoryVisibilityForRoom(authEvents) // 1. If the history_visibility was set to world_readable, allow. if historyVisibility == "world_readable" { @@ -52,7 +52,7 @@ func IsServerAllowed( return false } -func historyVisibilityForRoom(authEvents []gomatrixserverlib.Event) string { +func HistoryVisibilityForRoom(authEvents []gomatrixserverlib.Event) string { // https://matrix.org/docs/spec/client_server/r0.6.0#id87 // By default if no history_visibility is set, or if the value is not understood, the visibility is assumed to be shared. visibility := "shared" diff --git a/roomserver/query/backfill.go b/roomserver/query/backfill.go index 09a515e99..f518de3e8 100644 --- a/roomserver/query/backfill.go +++ b/roomserver/query/backfill.go @@ -3,6 +3,7 @@ package query import ( "context" + "github.com/matrix-org/dendrite/roomserver/auth" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" @@ -62,9 +63,9 @@ FederationHit: logrus.WithField("event_id", targetEvent.EventID()).Info("Requesting /state_ids at event") for _, srv := range b.servers { // hit any valid server c := gomatrixserverlib.FederatedStateProvider{ - FedClient: b.fedClient, - AuthEventsOnly: false, - Server: srv, + FedClient: b.fedClient, + RememberAuthEvents: false, + Server: srv, } res, err := c.StateIDsBeforeEvent(ctx, targetEvent) if err != nil { @@ -114,7 +115,9 @@ func (b *backfillRequester) calculateNewStateIDs(targetEvent, prevEvent gomatrix return nil } -func (b *backfillRequester) StateBeforeEvent(ctx context.Context, roomVer gomatrixserverlib.RoomVersion, event gomatrixserverlib.HeaderedEvent, eventIDs []string) (map[string]*gomatrixserverlib.Event, error) { +func (b *backfillRequester) StateBeforeEvent(ctx context.Context, roomVer gomatrixserverlib.RoomVersion, + event gomatrixserverlib.HeaderedEvent, eventIDs []string) (map[string]*gomatrixserverlib.Event, error) { + // try to fetch the events from the database first events, err := b.ProvideEvents(roomVer, eventIDs) if err != nil { @@ -133,9 +136,9 @@ func (b *backfillRequester) StateBeforeEvent(ctx context.Context, roomVer gomatr } c := gomatrixserverlib.FederatedStateProvider{ - FedClient: b.fedClient, - AuthEventsOnly: false, - Server: b.servers[0], + FedClient: b.fedClient, + RememberAuthEvents: false, + Server: b.servers[0], } result, err := c.StateBeforeEvent(ctx, roomVer, event, eventIDs) if err != nil { @@ -160,18 +163,33 @@ func (b *backfillRequester) ServersAtEvent(ctx context.Context, roomID, eventID return } + stateEntries, err := stateBeforeEvent(ctx, b.db, NIDs[eventID]) + if err != nil { + logrus.WithField("event_id", eventID).WithError(err).Error("ServersAtEvent: failed to load state before event") + return + } + + // possibly return all joined servers depending on history visiblity + memberEventsFromVis, err := joinEventsFromHistoryVisibility(ctx, b.db, roomID, stateEntries) + if err != nil { + logrus.WithError(err).Error("ServersAtEvent: failed calculate servers from history visibility rules") + return + } + logrus.Infof("ServersAtEvent including %d current events from history visibility", len(memberEventsFromVis)) + // Retrieve all "m.room.member" state events of "join" membership, which // contains the list of users in the room before the event, therefore all // the servers in it at that moment. - events, err := getMembershipsBeforeEventNID(ctx, b.db, NIDs[eventID], true) + memberEvents, err := getMembershipsAtState(ctx, b.db, stateEntries, true) if err != nil { logrus.WithField("event_id", eventID).WithError(err).Error("ServersAtEvent: failed to get memberships before event") return } + memberEvents = append(memberEvents, memberEventsFromVis...) // Store the server names in a temporary map to avoid duplicates. serverSet := make(map[gomatrixserverlib.ServerName]bool) - for _, event := range events { + for _, event := range memberEvents { serverSet[event.Origin()] = true } for server := range serverSet { @@ -186,7 +204,9 @@ func (b *backfillRequester) ServersAtEvent(ctx context.Context, roomID, eventID // Backfill performs a backfill request to the given server. // https://matrix.org/docs/spec/server_server/latest#get-matrix-federation-v1-backfill-roomid -func (b *backfillRequester) Backfill(ctx context.Context, server gomatrixserverlib.ServerName, roomID string, fromEventIDs []string, limit int) (*gomatrixserverlib.Transaction, error) { +func (b *backfillRequester) Backfill(ctx context.Context, server gomatrixserverlib.ServerName, roomID string, + fromEventIDs []string, limit int) (*gomatrixserverlib.Transaction, error) { + tx, err := b.fedClient.Backfill(ctx, server, roomID, limit, fromEventIDs) return &tx, err } @@ -215,3 +235,44 @@ func (b *backfillRequester) ProvideEvents(roomVer gomatrixserverlib.RoomVersion, } return events, nil } + +// joinEventsFromHistoryVisibility returns all CURRENTLY joined members if the provided state indicated a 'shared' history visibility. +// TODO: Long term we probably want a history_visibility table which stores eventNID | visibility_enum so we can just +// pull all events and then filter by that table. +func joinEventsFromHistoryVisibility( + ctx context.Context, db storage.Database, roomID string, stateEntries []types.StateEntry) ([]types.Event, error) { + + var eventNIDs []types.EventNID + for _, entry := range stateEntries { + // Filter the events to retrieve to only keep the membership events + if entry.EventTypeNID == types.MRoomHistoryVisibilityNID && entry.EventStateKeyNID == types.EmptyStateKeyNID { + eventNIDs = append(eventNIDs, entry.EventNID) + break + } + } + + // Get all of the events in this state + stateEvents, err := db.Events(ctx, eventNIDs) + if err != nil { + return nil, err + } + events := make([]gomatrixserverlib.Event, len(stateEvents)) + for i := range stateEvents { + events[i] = stateEvents[i].Event + } + visibility := auth.HistoryVisibilityForRoom(events) + if visibility != "shared" { + logrus.Infof("ServersAtEvent history visibility not shared: %s", visibility) + return nil, nil + } + // get joined members + roomNID, err := db.RoomNID(ctx, roomID) + if err != nil { + return nil, err + } + joinEventNIDs, err := db.GetMembershipEventNIDsForRoom(ctx, roomNID, true) + if err != nil { + return nil, err + } + return db.Events(ctx, joinEventNIDs) +} diff --git a/roomserver/query/query.go b/roomserver/query/query.go index a54fa58d9..6778ac280 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -277,6 +277,7 @@ func (r *RoomserverQueryAPI) QueryMembershipsForRoom( response.JoinEvents = []gomatrixserverlib.ClientEvent{} var events []types.Event + var stateEntries []types.StateEntry if stillInRoom { var eventNIDs []types.EventNID eventNIDs, err = r.DB.GetMembershipEventNIDsForRoom(ctx, roomNID, request.JoinedOnly) @@ -286,7 +287,12 @@ func (r *RoomserverQueryAPI) QueryMembershipsForRoom( events, err = r.DB.Events(ctx, eventNIDs) } else { - events, err = getMembershipsBeforeEventNID(ctx, r.DB, membershipEventNID, request.JoinedOnly) + stateEntries, err = stateBeforeEvent(ctx, r.DB, membershipEventNID) + if err != nil { + logrus.WithField("membership_event_nid", membershipEventNID).WithError(err).Error("failed to load state before event") + return err + } + events, err = getMembershipsAtState(ctx, r.DB, stateEntries, request.JoinedOnly) } if err != nil { @@ -301,15 +307,8 @@ func (r *RoomserverQueryAPI) QueryMembershipsForRoom( return nil } -// getMembershipsBeforeEventNID takes the numeric ID of an event and fetches the state -// of the event's room as it was when this event was fired, then filters the state events to -// only keep the "m.room.member" events with a "join" membership. These events are returned. -// Returns an error if there was an issue fetching the events. -func getMembershipsBeforeEventNID( - ctx context.Context, db storage.Database, eventNID types.EventNID, joinedOnly bool, -) ([]types.Event, error) { +func stateBeforeEvent(ctx context.Context, db storage.Database, eventNID types.EventNID) ([]types.StateEntry, error) { roomState := state.NewStateResolution(db) - events := []types.Event{} // Lookup the event NID eIDs, err := db.EventIDs(ctx, []types.EventNID{eventNID}) if err != nil { @@ -323,10 +322,15 @@ func getMembershipsBeforeEventNID( } // Fetch the state as it was when this event was fired - stateEntries, err := roomState.LoadCombinedStateAfterEvents(ctx, prevState) - if err != nil { - return nil, err - } + return roomState.LoadCombinedStateAfterEvents(ctx, prevState) +} + +// getMembershipsAtState filters the state events to +// only keep the "m.room.member" events with a "join" membership. These events are returned. +// Returns an error if there was an issue fetching the events. +func getMembershipsAtState( + ctx context.Context, db storage.Database, stateEntries []types.StateEntry, joinedOnly bool, +) ([]types.Event, error) { var eventNIDs []types.EventNID for _, entry := range stateEntries { @@ -347,6 +351,7 @@ func getMembershipsBeforeEventNID( } // Filter the events to only keep the "join" membership events + var events []types.Event for _, event := range stateEvents { membership, err := event.Membership() if err != nil { @@ -563,20 +568,29 @@ func (r *RoomserverQueryAPI) backfillViaFederation(ctx context.Context, req *api if !ok { // this should be impossible as all events returned must have pass Step 5 of the PDU checks // which requires a list of state IDs. - logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to find state IDs for event which passed auth checks") + logrus.WithError(err).WithField("event_id", ev.EventID()).Error("backfillViaFederation: failed to find state IDs for event which passed auth checks") continue } var entries []types.StateEntry if entries, err = r.DB.StateEntriesForEventIDs(ctx, stateIDs); err != nil { - return err + // attempt to fetch the missing events + r.fetchAndStoreMissingEvents(ctx, roomVer, requester, stateIDs) + // try again + entries, err = r.DB.StateEntriesForEventIDs(ctx, stateIDs) + if err != nil { + logrus.WithError(err).WithField("event_id", ev.EventID()).Error("backfillViaFederation: failed to get state entries for event") + return err + } } var beforeStateSnapshotNID types.StateSnapshotNID if beforeStateSnapshotNID, err = r.DB.AddState(ctx, roomNID, nil, entries); err != nil { + logrus.WithError(err).WithField("event_id", ev.EventID()).Error("backfillViaFederation: failed to persist state entries to get snapshot nid") return err } + util.GetLogger(ctx).Infof("Backfilled event %s (nid=%d) getting snapshot %v with entries %+v", ev.EventID(), ev.EventNID, beforeStateSnapshotNID, entries) if err = r.DB.SetState(ctx, ev.EventNID, beforeStateSnapshotNID); err != nil { - logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to set state before event") + logrus.WithError(err).WithField("event_id", ev.EventID()).Error("backfillViaFederation: failed to persist snapshot nid") } } @@ -608,6 +622,66 @@ func (r *RoomserverQueryAPI) isServerCurrentlyInRoom(ctx context.Context, server return auth.IsAnyUserOnServerWithMembership(serverName, gmslEvents, gomatrixserverlib.Join), nil } +// fetchAndStoreMissingEvents does a best-effort fetch and store of missing events specified in stateIDs. Returns no error as it is just +// best effort. +func (r *RoomserverQueryAPI) fetchAndStoreMissingEvents(ctx context.Context, roomVer gomatrixserverlib.RoomVersion, + backfillRequester *backfillRequester, stateIDs []string) { + + servers := backfillRequester.servers + + // work out which are missing + nidMap, err := r.DB.EventNIDs(ctx, stateIDs) + if err != nil { + util.GetLogger(ctx).WithError(err).Warn("cannot query missing events") + return + } + missingMap := make(map[string]*gomatrixserverlib.HeaderedEvent) // id -> event + for _, id := range stateIDs { + if _, ok := nidMap[id]; !ok { + missingMap[id] = nil + } + } + util.GetLogger(ctx).Infof("Fetching %d missing state events (from %d possible servers)", len(missingMap), len(servers)) + + // fetch the events from federation. Loop the servers first so if we find one that works we stick with them + for _, srv := range servers { + for id, ev := range missingMap { + if ev != nil { + continue // already found + } + logger := util.GetLogger(ctx).WithField("server", srv).WithField("event_id", id) + res, err := r.FedClient.GetEvent(ctx, srv, id) + if err != nil { + logger.WithError(err).Warn("failed to get event from server") + continue + } + loader := gomatrixserverlib.NewEventsLoader(roomVer, r.KeyRing, backfillRequester, backfillRequester.ProvideEvents, false) + result, err := loader.LoadAndVerify(ctx, res.PDUs, gomatrixserverlib.TopologicalOrderByPrevEvents) + if err != nil { + logger.WithError(err).Warn("failed to load and verify event") + continue + } + logger.Infof("returned %d PDUs which made events %+v", len(res.PDUs), result) + for _, res := range result { + if res.Error != nil { + logger.WithError(err).Warn("event failed PDU checks") + continue + } + missingMap[id] = res.Event + } + } + } + + var newEvents []gomatrixserverlib.HeaderedEvent + for _, ev := range missingMap { + if ev != nil { + newEvents = append(newEvents, *ev) + } + } + util.GetLogger(ctx).Infof("Persisting %d new events", len(newEvents)) + persistEvents(ctx, r.DB, newEvents) +} + // TODO: Remove this when we have tests to assert correctness of this function // nolint:gocyclo func (r *RoomserverQueryAPI) scanEventTree( @@ -857,7 +931,7 @@ func persistEvents(ctx context.Context, db storage.Database, events []gomatrixse var stateAtEvent types.StateAtEvent roomNID, stateAtEvent, err = db.StoreEvent(ctx, ev.Unwrap(), nil, authNids) if err != nil { - logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to store backfilled event") + logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to persist event") continue } backfilledEventMap[ev.EventID()] = types.Event{ diff --git a/roomserver/state/state.go b/roomserver/state/state.go index 389c94400..9b005ee6a 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -86,7 +86,10 @@ func (v StateResolution) LoadStateAtEvent( ) ([]types.StateEntry, error) { snapshotNID, err := v.db.SnapshotNIDFromEventID(ctx, eventID) if err != nil { - return nil, err + return nil, fmt.Errorf("LoadStateAtEvent.SnapshotNIDFromEventID failed for event %s : %s", eventID, err) + } + if snapshotNID == 0 { + return nil, fmt.Errorf("LoadStateAtEvent.SnapshotNIDFromEventID(%s) returned 0 NID, was this event stored?", eventID) } stateEntries, err := v.LoadStateAtSnapshot(ctx, snapshotNID) diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index d881fa91f..a63596aeb 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -48,11 +48,6 @@ const insertEventSQL = ` ON CONFLICT DO NOTHING; ` -const insertEventResultSQL = ` - SELECT event_nid, state_snapshot_nid FROM roomserver_events - WHERE rowid = last_insert_rowid(); -` - const selectEventSQL = "" + "SELECT event_nid, state_snapshot_nid FROM roomserver_events WHERE event_id = $1" @@ -102,7 +97,6 @@ const selectRoomNIDForEventNIDSQL = "" + type eventStatements struct { db *sql.DB insertEventStmt *sql.Stmt - insertEventResultStmt *sql.Stmt selectEventStmt *sql.Stmt bulkSelectStateEventByIDStmt *sql.Stmt bulkSelectStateAtEventByIDStmt *sql.Stmt @@ -126,7 +120,6 @@ func (s *eventStatements) prepare(db *sql.DB) (err error) { return statementList{ {&s.insertEventStmt, insertEventSQL}, - {&s.insertEventResultStmt, insertEventResultSQL}, {&s.selectEventStmt, selectEventSQL}, {&s.bulkSelectStateEventByIDStmt, bulkSelectStateEventByIDSQL}, {&s.bulkSelectStateAtEventByIDStmt, bulkSelectStateAtEventByIDSQL}, @@ -152,19 +145,22 @@ func (s *eventStatements) insertEvent( referenceSHA256 []byte, authEventNIDs []types.EventNID, depth int64, -) (types.EventNID, types.StateSnapshotNID, error) { - var eventNID int64 - var stateNID int64 - var err error +) (types.EventNID, error) { + // attempt to insert: the last_row_id is the event NID insertStmt := common.TxStmt(txn, s.insertEventStmt) - resultStmt := common.TxStmt(txn, s.insertEventResultStmt) - if _, err = insertStmt.ExecContext( + result, err := insertStmt.ExecContext( ctx, int64(roomNID), int64(eventTypeNID), int64(eventStateKeyNID), eventID, referenceSHA256, eventNIDsAsArray(authEventNIDs), depth, - ); err == nil { - err = resultStmt.QueryRowContext(ctx).Scan(&eventNID, &stateNID) + ) + if err != nil { + return 0, err } - return types.EventNID(eventNID), types.StateSnapshotNID(stateNID), err + modified, err := result.RowsAffected() + if modified == 0 && err == nil { + return 0, sql.ErrNoRows + } + eventNID, err := result.LastInsertId() + return types.EventNID(eventNID), err } func (s *eventStatements) selectEvent( diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index 5df9c4e08..b6e846df7 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -124,7 +124,7 @@ func (d *Database) StoreEvent( } } - if eventNID, stateNID, err = d.statements.insertEvent( + if eventNID, err = d.statements.insertEvent( ctx, txn, roomNID, From 77fe509031d578a133e648a9eae3dd727eb2aeb5 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 29 Apr 2020 19:37:00 +0100 Subject: [PATCH 08/12] Enable v5 rooms (#992) * Enable v5 roooms * Update sytest-whitelist * Enable v5 rooms by default, update gomatrixserverlib --- go.mod | 2 +- go.sum | 4 ++-- roomserver/version/version.go | 6 +++--- sytest-whitelist | 6 ++++++ 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index d20089e8a..7cda4fe31 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200429162354-392f0b1b7421 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200429173835-c4e77c0d7e72 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index da0e02d6f..5b83ed09d 100644 --- a/go.sum +++ b/go.sum @@ -367,8 +367,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200429162354-392f0b1b7421 h1:4zP29YlpfEtJ9a7sZ33Mf0FJInD2N3/KzDcLa62bRKc= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200429162354-392f0b1b7421/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200429173835-c4e77c0d7e72 h1:onwidlObCIqpqXpaqU2BISW4Ngq4ETer9AsbCTiR8T4= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200429173835-c4e77c0d7e72/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= diff --git a/roomserver/version/version.go b/roomserver/version/version.go index f2a67e74d..ddd0f23a6 100644 --- a/roomserver/version/version.go +++ b/roomserver/version/version.go @@ -51,15 +51,15 @@ var roomVersions = map[gomatrixserverlib.RoomVersion]RoomVersionDescription{ Stable: true, }, gomatrixserverlib.RoomVersionV5: RoomVersionDescription{ - Supported: false, - Stable: false, + Supported: true, + Stable: true, }, } // DefaultRoomVersion contains the room version that will, by // default, be used to create new rooms on this server. func DefaultRoomVersion() gomatrixserverlib.RoomVersion { - return gomatrixserverlib.RoomVersionV4 + return gomatrixserverlib.RoomVersionV5 } // RoomVersions returns a map of all known room versions to this diff --git a/sytest-whitelist b/sytest-whitelist index 439c306c7..c957021cb 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -258,3 +258,9 @@ User can invite remote user to room with version 1 User can invite remote user to room with version 2 User can invite remote user to room with version 3 User can invite remote user to room with version 4 +User can create and send/receive messages in a room with version 5 +local user can join room with version 5 +User can invite local user to room with version 5 +remote user can join room with version 5 +User can invite remote user to room with version 5 +Remote user can backfill in a room with version 5 From 540f6fcd9475a9c562837d59cf008ee6d0ce8983 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 30 Apr 2020 13:50:11 +0100 Subject: [PATCH 09/12] Update gmsl for key validity fix --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7cda4fe31..55c1179c2 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200429173835-c4e77c0d7e72 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200430104311-8d41c4d924ec github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index 5b83ed09d..8c1d15ad7 100644 --- a/go.sum +++ b/go.sum @@ -367,8 +367,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200429173835-c4e77c0d7e72 h1:onwidlObCIqpqXpaqU2BISW4Ngq4ETer9AsbCTiR8T4= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200429173835-c4e77c0d7e72/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200430104311-8d41c4d924ec h1:9MvZSZzBKvCWqM5KXMGZ1PBDrSLcxs5zfc561UPgcYA= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200430104311-8d41c4d924ec/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= From ebbfc125920beb321713e28a2a137d768406fa15 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 30 Apr 2020 17:15:29 +0100 Subject: [PATCH 10/12] Add tests for the storage interface (#995) * Move docs to interface * Add tests around syncing * Add topology token test * Linting --- syncapi/storage/interface.go | 59 ++++- syncapi/storage/postgres/syncserver.go | 54 ----- syncapi/storage/storage_test.go | 313 +++++++++++++++++++++++++ 3 files changed, 371 insertions(+), 55 deletions(-) create mode 100644 syncapi/storage/storage_test.go diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index a3efd8d58..bd9504db8 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -28,26 +28,83 @@ import ( type Database interface { common.PartitionStorer + // AllJoinedUsersInRooms returns a map of room ID to a list of all joined user IDs. AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) + // Events lookups a list of event by their event ID. + // Returns a list of events matching the requested IDs found in the database. + // If an event is not found in the database then it will be omitted from the list. + // Returns an error if there was a problem talking with the database. + // Does not include any transaction IDs in the returned events. Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error) - WriteEvent(context.Context, *gomatrixserverlib.HeaderedEvent, []gomatrixserverlib.HeaderedEvent, []string, []string, *api.TransactionID, bool) (types.StreamPosition, error) + // WriteEvent into the database. It is not safe to call this function from multiple goroutines, as it would create races + // when generating the sync stream position for this event. Returns the sync stream position for the inserted event. + // Returns an error if there was a problem inserting this event. + WriteEvent(ctx context.Context, ev *gomatrixserverlib.HeaderedEvent, addStateEvents []gomatrixserverlib.HeaderedEvent, + addStateEventIDs []string, removeStateEventIDs []string, transactionID *api.TransactionID, excludeFromSync bool) (types.StreamPosition, error) + // GetStateEvent returns the Matrix state event of a given type for a given room with a given state key + // If no event could be found, returns nil + // If there was an issue during the retrieval, returns an error GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error) + // GetStateEventsForRoom fetches the state events for a given room. + // Returns an empty slice if no state events could be found for this room. + // Returns an error if there was an issue with the retrieval. GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter) (stateEvents []gomatrixserverlib.HeaderedEvent, err error) + // SyncPosition returns the latest positions for syncing. SyncPosition(ctx context.Context) (types.PaginationToken, error) + // IncrementalSync returns all the data needed in order to create an incremental + // sync response for the given user. Events returned will include any client + // transaction IDs associated with the given device. These transaction IDs come + // from when the device sent the event via an API that included a transaction + // ID. IncrementalSync(ctx context.Context, device authtypes.Device, fromPos, toPos types.PaginationToken, numRecentEventsPerRoom int, wantFullState bool) (*types.Response, error) + // CompleteSync returns a complete /sync API response for the given user. CompleteSync(ctx context.Context, userID string, numRecentEventsPerRoom int) (*types.Response, error) + // GetAccountDataInRange returns all account data for a given user inserted or + // updated between two given positions + // Returns a map following the format data[roomID] = []dataTypes + // If no data is retrieved, returns an empty map + // If there was an issue with the retrieval, returns an error GetAccountDataInRange(ctx context.Context, userID string, oldPos, newPos types.StreamPosition, accountDataFilterPart *gomatrixserverlib.EventFilter) (map[string][]string, error) + // UpsertAccountData keeps track of new or updated account data, by saving the type + // of the new/updated data, and the user ID and room ID the data is related to (empty) + // room ID means the data isn't specific to any room) + // If no data with the given type, user ID and room ID exists in the database, + // creates a new row, else update the existing one + // Returns an error if there was an issue with the upsert UpsertAccountData(ctx context.Context, userID, roomID, dataType string) (types.StreamPosition, error) + // AddInviteEvent stores a new invite event for a user. + // If the invite was successfully stored this returns the stream ID it was stored at. + // Returns an error if there was a problem communicating with the database. AddInviteEvent(ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent) (types.StreamPosition, error) + // RetireInviteEvent removes an old invite event from the database. + // Returns an error if there was a problem communicating with the database. RetireInviteEvent(ctx context.Context, inviteEventID string) error + // SetTypingTimeoutCallback sets a callback function that is called right after + // a user is removed from the typing user list due to timeout. SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) + // AddTypingUser adds a typing user to the typing cache. + // Returns the newly calculated sync position for typing notifications. AddTypingUser(userID, roomID string, expireTime *time.Time) types.StreamPosition + // RemoveTypingUser removes a typing user from the typing cache. + // Returns the newly calculated sync position for typing notifications. RemoveTypingUser(userID, roomID string) types.StreamPosition + // GetEventsInRange retrieves all of the events on a given ordering using the + // given extremities and limit. GetEventsInRange(ctx context.Context, from, to *types.PaginationToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error) + // EventPositionInTopology returns the depth of the given event. EventPositionInTopology(ctx context.Context, eventID string) (types.StreamPosition, error) + // EventsAtTopologicalPosition returns all of the events matching a given + // position in the topology of a given room. EventsAtTopologicalPosition(ctx context.Context, roomID string, pos types.StreamPosition) ([]types.StreamEvent, error) + // BackwardExtremitiesForRoom returns the event IDs of all of the backward + // extremities we know of for a given room. BackwardExtremitiesForRoom(ctx context.Context, roomID string) (backwardExtremities []string, err error) + // MaxTopologicalPosition returns the highest topological position for a given room. MaxTopologicalPosition(ctx context.Context, roomID string) (types.StreamPosition, error) + // StreamEventsToEvents converts streamEvent to Event. If device is non-nil and + // matches the streamevent.transactionID device then the transaction ID gets + // added to the unsigned section of the output event. StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent + // SyncStreamPosition returns the latest position in the sync stream. Returns 0 if there are no events yet. SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) } diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index 9d61ccfc3..ef9707397 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -93,16 +93,10 @@ func NewSyncServerDatasource(dbDataSourceName string) (*SyncServerDatasource, er return &d, nil } -// AllJoinedUsersInRooms returns a map of room ID to a list of all joined user IDs. func (d *SyncServerDatasource) AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) { return d.roomstate.selectJoinedUsers(ctx) } -// Events lookups a list of event by their event ID. -// Returns a list of events matching the requested IDs found in the database. -// If an event is not found in the database then it will be omitted from the list. -// Returns an error if there was a problem talking with the database. -// Does not include any transaction IDs in the returned events. func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error) { streamEvents, err := d.events.selectEvents(ctx, nil, eventIDs) if err != nil { @@ -148,9 +142,6 @@ func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, tx return nil } -// WriteEvent into the database. It is not safe to call this function from multiple goroutines, as it would create races -// when generating the sync stream position for this event. Returns the sync stream position for the inserted event. -// Returns an error if there was a problem inserting this event. func (d *SyncServerDatasource) WriteEvent( ctx context.Context, ev *gomatrixserverlib.HeaderedEvent, @@ -221,18 +212,12 @@ func (d *SyncServerDatasource) updateRoomState( return nil } -// GetStateEvent returns the Matrix state event of a given type for a given room with a given state key -// If no event could be found, returns nil -// If there was an issue during the retrieval, returns an error func (d *SyncServerDatasource) GetStateEvent( ctx context.Context, roomID, evType, stateKey string, ) (*gomatrixserverlib.HeaderedEvent, error) { return d.roomstate.selectStateEvent(ctx, roomID, evType, stateKey) } -// GetStateEventsForRoom fetches the state events for a given room. -// Returns an empty slice if no state events could be found for this room. -// Returns an error if there was an issue with the retrieval. func (d *SyncServerDatasource) GetStateEventsForRoom( ctx context.Context, roomID string, stateFilter *gomatrixserverlib.StateFilter, ) (stateEvents []gomatrixserverlib.HeaderedEvent, err error) { @@ -243,8 +228,6 @@ func (d *SyncServerDatasource) GetStateEventsForRoom( return } -// GetEventsInRange retrieves all of the events on a given ordering using the -// given extremities and limit. func (d *SyncServerDatasource) GetEventsInRange( ctx context.Context, from, to *types.PaginationToken, @@ -306,29 +289,22 @@ func (d *SyncServerDatasource) GetEventsInRange( return } -// SyncPosition returns the latest positions for syncing. func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (types.PaginationToken, error) { return d.syncPositionTx(ctx, nil) } -// BackwardExtremitiesForRoom returns the event IDs of all of the backward -// extremities we know of for a given room. func (d *SyncServerDatasource) BackwardExtremitiesForRoom( ctx context.Context, roomID string, ) (backwardExtremities []string, err error) { return d.backwardExtremities.SelectBackwardExtremitiesForRoom(ctx, roomID) } -// MaxTopologicalPosition returns the highest topological position for a given -// room. func (d *SyncServerDatasource) MaxTopologicalPosition( ctx context.Context, roomID string, ) (types.StreamPosition, error) { return d.topology.selectMaxPositionInTopology(ctx, roomID) } -// EventsAtTopologicalPosition returns all of the events matching a given -// position in the topology of a given room. func (d *SyncServerDatasource) EventsAtTopologicalPosition( ctx context.Context, roomID string, pos types.StreamPosition, ) ([]types.StreamEvent, error) { @@ -346,7 +322,6 @@ func (d *SyncServerDatasource) EventPositionInTopology( return d.topology.selectPositionInTopology(ctx, eventID) } -// SyncStreamPosition returns the latest position in the sync stream. Returns 0 if there are no events yet. func (d *SyncServerDatasource) SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) { return d.syncStreamPositionTx(ctx, nil) } @@ -511,11 +486,6 @@ func (d *SyncServerDatasource) addEDUDeltaToResponse( return } -// IncrementalSync returns all the data needed in order to create an incremental -// sync response for the given user. Events returned will include any client -// transaction IDs associated with the given device. These transaction IDs come -// from when the device sent the event via an API that included a transaction -// ID. func (d *SyncServerDatasource) IncrementalSync( ctx context.Context, device authtypes.Device, @@ -645,7 +615,6 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( return res, toPos, joinedRoomIDs, err } -// CompleteSync returns a complete /sync API response for the given user. func (d *SyncServerDatasource) CompleteSync( ctx context.Context, userID string, numRecentEventsPerRoom int, ) (*types.Response, error) { @@ -677,11 +646,6 @@ var txReadOnlySnapshot = sql.TxOptions{ ReadOnly: true, } -// GetAccountDataInRange returns all account data for a given user inserted or -// updated between two given positions -// Returns a map following the format data[roomID] = []dataTypes -// If no data is retrieved, returns an empty map -// If there was an issue with the retrieval, returns an error func (d *SyncServerDatasource) GetAccountDataInRange( ctx context.Context, userID string, oldPos, newPos types.StreamPosition, accountDataFilterPart *gomatrixserverlib.EventFilter, @@ -689,29 +653,18 @@ func (d *SyncServerDatasource) GetAccountDataInRange( return d.accountData.selectAccountDataInRange(ctx, userID, oldPos, newPos, accountDataFilterPart) } -// UpsertAccountData keeps track of new or updated account data, by saving the type -// of the new/updated data, and the user ID and room ID the data is related to (empty) -// room ID means the data isn't specific to any room) -// If no data with the given type, user ID and room ID exists in the database, -// creates a new row, else update the existing one -// Returns an error if there was an issue with the upsert func (d *SyncServerDatasource) UpsertAccountData( ctx context.Context, userID, roomID, dataType string, ) (types.StreamPosition, error) { return d.accountData.insertAccountData(ctx, userID, roomID, dataType) } -// AddInviteEvent stores a new invite event for a user. -// If the invite was successfully stored this returns the stream ID it was stored at. -// Returns an error if there was a problem communicating with the database. func (d *SyncServerDatasource) AddInviteEvent( ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, ) (types.StreamPosition, error) { return d.invites.insertInviteEvent(ctx, inviteEvent) } -// RetireInviteEvent removes an old invite event from the database. -// Returns an error if there was a problem communicating with the database. func (d *SyncServerDatasource) RetireInviteEvent( ctx context.Context, inviteEventID string, ) error { @@ -725,16 +678,12 @@ func (d *SyncServerDatasource) SetTypingTimeoutCallback(fn cache.TimeoutCallback d.eduCache.SetTimeoutCallback(fn) } -// AddTypingUser adds a typing user to the typing cache. -// Returns the newly calculated sync position for typing notifications. func (d *SyncServerDatasource) AddTypingUser( userID, roomID string, expireTime *time.Time, ) types.StreamPosition { return types.StreamPosition(d.eduCache.AddTypingUser(userID, roomID, expireTime)) } -// RemoveTypingUser removes a typing user from the typing cache. -// Returns the newly calculated sync position for typing notifications. func (d *SyncServerDatasource) RemoveTypingUser( userID, roomID string, ) types.StreamPosition { @@ -1073,9 +1022,6 @@ func (d *SyncServerDatasource) currentStateStreamEventsForRoom( return s, nil } -// StreamEventsToEvents converts streamEvent to Event. If device is non-nil and -// matches the streamevent.transactionID device then the transaction ID gets -// added to the unsigned section of the output event. func (d *SyncServerDatasource) StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent { out := make([]gomatrixserverlib.HeaderedEvent, len(in)) for i := 0; i < len(in); i++ { diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go new file mode 100644 index 000000000..e591e7ed7 --- /dev/null +++ b/syncapi/storage/storage_test.go @@ -0,0 +1,313 @@ +package storage_test + +import ( + "context" + "crypto/ed25519" + "fmt" + "testing" + "time" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/gomatrixserverlib" +) + +var ( + ctx = context.Background() + emptyStateKey = "" + testOrigin = gomatrixserverlib.ServerName("hollow.knight") + testRoomID = fmt.Sprintf("!hallownest:%s", testOrigin) + testUserIDA = fmt.Sprintf("@hornet:%s", testOrigin) + testUserIDB = fmt.Sprintf("@paleking:%s", testOrigin) + testUserDeviceA = authtypes.Device{ + UserID: testUserIDA, + ID: "device_id_A", + DisplayName: "Device A", + } + testRoomVersion = gomatrixserverlib.RoomVersionV4 + testKeyID = gomatrixserverlib.KeyID("ed25519:storage_test") + testPrivateKey = ed25519.NewKeyFromSeed([]byte{ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + }) +) + +func MustCreateEvent(t *testing.T, roomID string, prevs []gomatrixserverlib.HeaderedEvent, b *gomatrixserverlib.EventBuilder) gomatrixserverlib.HeaderedEvent { + b.RoomID = roomID + if prevs != nil { + prevIDs := make([]string, len(prevs)) + for i := range prevs { + prevIDs[i] = prevs[i].EventID() + } + b.PrevEvents = prevIDs + } + e, err := b.Build(time.Now(), testOrigin, testKeyID, testPrivateKey, testRoomVersion) + if err != nil { + t.Fatalf("failed to build event: %s", err) + } + return e.Headered(testRoomVersion) +} + +func MustCreateDatabase(t *testing.T) storage.Database { + db, err := sqlite3.NewSyncServerDatasource("file::memory:") + if err != nil { + t.Fatalf("NewSyncServerDatasource returned %s", err) + } + return db +} + +// Create a list of events which include a create event, join event and some messages. +func SimpleRoom(t *testing.T, roomID, userA, userB string) (msgs []gomatrixserverlib.HeaderedEvent, state []gomatrixserverlib.HeaderedEvent) { + var events []gomatrixserverlib.HeaderedEvent + events = append(events, MustCreateEvent(t, roomID, nil, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"room_version":"4","creator":"%s"}`, userA)), + Type: "m.room.create", + StateKey: &emptyStateKey, + Sender: userA, + Depth: int64(len(events) + 1), + })) + state = append(state, events[len(events)-1]) + events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"membership":"join"}`)), + Type: "m.room.member", + StateKey: &userA, + Sender: userA, + Depth: int64(len(events) + 1), + })) + state = append(state, events[len(events)-1]) + for i := 0; i < 10; i++ { + events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"body":"Message A %d"}`, i+1)), + Type: "m.room.message", + Sender: userA, + Depth: int64(len(events) + 1), + })) + } + events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"membership":"join"}`)), + Type: "m.room.member", + StateKey: &userB, + Sender: userB, + Depth: int64(len(events) + 1), + })) + state = append(state, events[len(events)-1]) + for i := 0; i < 10; i++ { + events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"body":"Message B %d"}`, i+1)), + Type: "m.room.message", + Sender: userB, + Depth: int64(len(events) + 1), + })) + } + + return events, state +} + +func MustWriteEvents(t *testing.T, db storage.Database, events []gomatrixserverlib.HeaderedEvent) (positions []types.StreamPosition) { + for _, ev := range events { + var addStateEvents []gomatrixserverlib.HeaderedEvent + var addStateEventIDs []string + var removeStateEventIDs []string + if ev.StateKey() != nil { + addStateEvents = append(addStateEvents, ev) + addStateEventIDs = append(addStateEventIDs, ev.EventID()) + } + pos, err := db.WriteEvent(ctx, &ev, addStateEvents, addStateEventIDs, removeStateEventIDs, nil, false) + if err != nil { + t.Fatalf("WriteEvent failed: %s", err) + } + positions = append(positions, pos) + } + return +} + +func TestWriteEvents(t *testing.T) { + t.Parallel() + db := MustCreateDatabase(t) + events, _ := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) + MustWriteEvents(t, db, events) +} + +// These tests assert basic functionality of the IncrementalSync and CompleteSync functions. +func TestSyncResponse(t *testing.T) { + t.Parallel() + db := MustCreateDatabase(t) + events, state := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) + positions := MustWriteEvents(t, db, events) + latest, err := db.SyncPosition(ctx) + if err != nil { + t.Fatalf("failed to get SyncPosition: %s", err) + } + + testCases := []struct { + Name string + DoSync func() (*types.Response, error) + WantTimeline []gomatrixserverlib.HeaderedEvent + WantState []gomatrixserverlib.HeaderedEvent + }{ + // The purpose of this test is to make sure that incremental syncs are including up to the latest events. + // It's a basic sanity test that sync works. It creates a `since` token that is on the penultimate event. + // It makes sure the response includes the final event. + { + Name: "IncrementalSync penultimate", + DoSync: func() (*types.Response, error) { + from := types.NewPaginationTokenFromTypeAndPosition( // pretend we are at the penultimate event + types.PaginationTokenTypeStream, positions[len(positions)-2], types.StreamPosition(0), + ) + return db.IncrementalSync(ctx, testUserDeviceA, *from, latest, 5, false) + }, + WantTimeline: events[len(events)-1:], + }, + // The purpose of this test is to check that passing a `numRecentEventsPerRoom` correctly limits the + // number of returned events. This is critical for big rooms hence the test here. + { + Name: "IncrementalSync limited", + DoSync: func() (*types.Response, error) { + from := types.NewPaginationTokenFromTypeAndPosition( // pretend we are 10 events behind + types.PaginationTokenTypeStream, positions[len(positions)-11], types.StreamPosition(0), + ) + // limit is set to 5 + return db.IncrementalSync(ctx, testUserDeviceA, *from, latest, 5, false) + }, + // want the last 5 events, NOT the last 10. + WantTimeline: events[len(events)-5:], + }, + // The purpose of this test is to check that CompleteSync returns all the current state as well as + // honouring the `numRecentEventsPerRoom` value + { + Name: "CompleteSync limited", + DoSync: func() (*types.Response, error) { + // limit set to 5 + return db.CompleteSync(ctx, testUserIDA, 5) + }, + // want the last 5 events, NOT the last 10. + WantTimeline: events[len(events)-5:], + // want all state for the room + WantState: state, + }, + // The purpose of this test is to check that CompleteSync can return everything with a high enough + // `numRecentEventsPerRoom`. + { + Name: "CompleteSync", + DoSync: func() (*types.Response, error) { + return db.CompleteSync(ctx, testUserIDA, len(events)+1) + }, + WantTimeline: events, + // We want no state at all as that field in /sync is the delta between the token (beginning of time) + // and the START of the timeline. + }, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(st *testing.T) { + res, err := tc.DoSync() + if err != nil { + st.Fatalf("failed to do sync: %s", err) + } + next := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeStream, latest.PDUPosition, latest.EDUTypingPosition) + if res.NextBatch != next.String() { + st.Errorf("NextBatch got %s want %s", res.NextBatch, next.String()) + } + roomRes, ok := res.Rooms.Join[testRoomID] + if !ok { + st.Fatalf("IncrementalSync response missing room %s - response: %+v", testRoomID, res) + } + assertEventsEqual(st, "state for "+testRoomID, false, roomRes.State.Events, tc.WantState) + assertEventsEqual(st, "timeline for "+testRoomID, false, roomRes.Timeline.Events, tc.WantTimeline) + }) + } +} + +// The purpose of this test is to ensure that backfill does indeed go backwards, using a stream token. +func TestGetEventsInRangeWithStreamToken(t *testing.T) { + t.Parallel() + db := MustCreateDatabase(t) + events, _ := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) + MustWriteEvents(t, db, events) + latest, err := db.SyncPosition(ctx) + if err != nil { + t.Fatalf("failed to get SyncPosition: %s", err) + } + // head towards the beginning of time + to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) + + // backpaginate 5 messages starting at the latest position. + paginatedEvents, err := db.GetEventsInRange(ctx, &latest, to, testRoomID, 5, true) + if err != nil { + t.Fatalf("GetEventsInRange returned an error: %s", err) + } + gots := gomatrixserverlib.HeaderedToClientEvents(db.StreamEventsToEvents(&testUserDeviceA, paginatedEvents), gomatrixserverlib.FormatAll) + assertEventsEqual(t, "", true, gots, reversed(events[len(events)-5:])) +} + +// The purpose of this test is to ensure that backfill does indeed go backwards, using a topology token +func TestGetEventsInRangeWithTopologyToken(t *testing.T) { + t.Parallel() + db := MustCreateDatabase(t) + events, _ := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) + MustWriteEvents(t, db, events) + latest, err := db.MaxTopologicalPosition(ctx, testRoomID) + if err != nil { + t.Fatalf("failed to get MaxTopologicalPosition: %s", err) + } + from := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, latest, 0) + // head towards the beginning of time + to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) + + // backpaginate 5 messages starting at the latest position. + paginatedEvents, err := db.GetEventsInRange(ctx, from, to, testRoomID, 5, true) + if err != nil { + t.Fatalf("GetEventsInRange returned an error: %s", err) + } + gots := gomatrixserverlib.HeaderedToClientEvents(db.StreamEventsToEvents(&testUserDeviceA, paginatedEvents), gomatrixserverlib.FormatAll) + assertEventsEqual(t, "", true, gots, reversed(events[len(events)-5:])) +} + +func assertEventsEqual(t *testing.T, msg string, checkRoomID bool, gots []gomatrixserverlib.ClientEvent, wants []gomatrixserverlib.HeaderedEvent) { + if len(gots) != len(wants) { + t.Fatalf("%s response returned %d events, want %d", msg, len(gots), len(wants)) + } + for i := range gots { + g := gots[i] + w := wants[i] + if g.EventID != w.EventID() { + t.Errorf("%s event[%d] event_id mismatch: got %s want %s", msg, i, g.EventID, w.EventID()) + } + if g.Sender != w.Sender() { + t.Errorf("%s event[%d] sender mismatch: got %s want %s", msg, i, g.Sender, w.Sender()) + } + if checkRoomID && g.RoomID != w.RoomID() { + t.Errorf("%s event[%d] room_id mismatch: got %s want %s", msg, i, g.RoomID, w.RoomID()) + } + if g.Type != w.Type() { + t.Errorf("%s event[%d] event type mismatch: got %s want %s", msg, i, g.Type, w.Type()) + } + if g.OriginServerTS != w.OriginServerTS() { + t.Errorf("%s event[%d] origin_server_ts mismatch: got %v want %v", msg, i, g.OriginServerTS, w.OriginServerTS()) + } + if string(g.Content) != string(w.Content()) { + t.Errorf("%s event[%d] content mismatch: got %s want %s", msg, i, string(g.Content), string(w.Content())) + } + if string(g.Unsigned) != string(w.Unsigned()) { + t.Errorf("%s event[%d] unsigned mismatch: got %s want %s", msg, i, string(g.Unsigned), string(w.Unsigned())) + } + if (g.StateKey == nil && w.StateKey() != nil) || (g.StateKey != nil && w.StateKey() == nil) { + t.Fatalf("%s event[%d] state_key [not] missing: got %v want %v", msg, i, g.StateKey, w.StateKey()) + } + if g.StateKey != nil { + if !w.StateKeyEquals(*g.StateKey) { + t.Errorf("%s event[%d] state_key mismatch: got %s want %s", msg, i, *g.StateKey, *w.StateKey()) + } + } + } +} + +func reversed(in []gomatrixserverlib.HeaderedEvent) []gomatrixserverlib.HeaderedEvent { + out := make([]gomatrixserverlib.HeaderedEvent, len(in)) + for i := 0; i < len(in); i++ { + out[i] = in[len(in)-i-1] + } + return out +} From e15f6676ac3f76ec2ef679c2df300d6a8e7e668f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 1 May 2020 10:48:17 +0100 Subject: [PATCH 11/12] Consolidation of roomserver APIs (#994) * Consolidation of roomserver APIs * Comment out alias tests for now, they are broken * Wire AS API into roomserver again * Roomserver didn't take asAPI param before so return to that * Prevent roomserver asking AS API for alias info * Rename some files * Remove alias_test, incoherent tests and unwanted appservice integration * Remove FS API inject on syncapi component --- appservice/appservice.go | 7 +- appservice/consumers/roomserver.go | 13 +- appservice/routing/routing.go | 2 +- clientapi/clientapi.go | 10 +- clientapi/consumers/roomserver.go | 22 +- clientapi/producers/roomserver.go | 12 +- clientapi/routing/capabilities.go | 4 +- clientapi/routing/createroom.go | 8 +- clientapi/routing/directory.go | 6 +- clientapi/routing/getevent.go | 6 +- clientapi/routing/joinroom.go | 14 +- clientapi/routing/membership.go | 16 +- clientapi/routing/memberships.go | 6 +- clientapi/routing/profile.go | 14 +- clientapi/routing/routing.go | 41 ++- clientapi/routing/sendevent.go | 10 +- clientapi/routing/state.go | 8 +- clientapi/threepid/invites.go | 8 +- cmd/dendrite-appservice-server/main.go | 4 +- cmd/dendrite-client-api-server/main.go | 5 +- cmd/dendrite-demo-libp2p/main.go | 27 +- cmd/dendrite-federation-api-server/main.go | 7 +- cmd/dendrite-federation-sender-server/main.go | 9 +- cmd/dendrite-monolith-server/main.go | 28 +- cmd/dendrite-public-rooms-api-server/main.go | 7 +- cmd/dendrite-room-server/main.go | 4 +- cmd/dendrite-sync-api-server/main.go | 4 +- cmd/dendritejs/main.go | 16 +- cmd/roomserver-integration-tests/main.go | 8 +- common/basecomponent/base.go | 20 +- common/events.go | 10 +- federationapi/federationapi.go | 10 +- federationapi/routing/backfill.go | 4 +- federationapi/routing/eventauth.go | 4 +- federationapi/routing/events.go | 10 +- federationapi/routing/join.go | 16 +- federationapi/routing/leave.go | 6 +- federationapi/routing/missingevents.go | 4 +- federationapi/routing/query.go | 4 +- federationapi/routing/routing.go | 29 +- federationapi/routing/send.go | 10 +- federationapi/routing/state.go | 14 +- federationapi/routing/threepid.go | 22 +- federationsender/consumers/roomserver.go | 26 +- federationsender/federationsender.go | 7 +- federationsender/producers/roomserver.go | 6 +- publicroomsapi/consumers/roomserver.go | 20 +- publicroomsapi/directory/directory.go | 6 +- publicroomsapi/publicroomsapi.go | 6 +- publicroomsapi/routing/routing.go | 4 +- roomserver/alias/alias_test.go | 209 ------------- roomserver/api/alias.go | 64 +--- roomserver/api/api.go | 141 +++++++++ roomserver/api/http.go | 41 +++ roomserver/api/input.go | 41 +-- roomserver/api/query.go | 131 +------- roomserver/{alias => internal}/alias.go | 186 ++++-------- roomserver/internal/api.go | 287 ++++++++++++++++++ roomserver/{input => internal}/input.go | 46 +-- .../input_authevents.go} | 2 +- .../input_authevents_test.go} | 2 +- .../events.go => internal/input_events.go} | 2 +- .../input_latest_events.go} | 2 +- .../input_membership.go} | 2 +- roomserver/{query => internal}/query.go | 250 ++------------- .../query_backfill.go} | 2 +- roomserver/{query => internal}/query_test.go | 6 +- roomserver/roomserver.go | 46 +-- syncapi/consumers/roomserver.go | 22 +- syncapi/routing/messages.go | 8 +- syncapi/routing/routing.go | 4 +- syncapi/syncapi.go | 6 +- 72 files changed, 894 insertions(+), 1170 deletions(-) delete mode 100644 roomserver/alias/alias_test.go create mode 100644 roomserver/api/api.go create mode 100644 roomserver/api/http.go rename roomserver/{alias => internal}/alias.go (50%) create mode 100644 roomserver/internal/api.go rename roomserver/{input => internal}/input.go (55%) rename roomserver/{input/authevents.go => internal/input_authevents.go} (99%) rename roomserver/{input/authevents_test.go => internal/input_authevents_test.go} (99%) rename roomserver/{input/events.go => internal/input_events.go} (99%) rename roomserver/{input/latest_events.go => internal/input_latest_events.go} (99%) rename roomserver/{input/membership.go => internal/input_membership.go} (99%) rename roomserver/{query => internal}/query.go (74%) rename roomserver/{query/backfill.go => internal/query_backfill.go} (99%) rename roomserver/{query => internal}/query_test.go (96%) diff --git a/appservice/appservice.go b/appservice/appservice.go index 181799879..71d131998 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -44,8 +44,7 @@ func SetupAppServiceAPIComponent( accountsDB accounts.Database, deviceDB devices.Database, federation *gomatrixserverlib.FederationClient, - roomserverAliasAPI roomserverAPI.RoomserverAliasAPI, - roomserverQueryAPI roomserverAPI.RoomserverQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, transactionsCache *transactions.Cache, ) appserviceAPI.AppServiceQueryAPI { // Create a connection to the appservice postgres DB @@ -87,7 +86,7 @@ func SetupAppServiceAPIComponent( consumer := consumers.NewOutputRoomEventConsumer( base.Cfg, base.KafkaConsumer, accountsDB, appserviceDB, - roomserverQueryAPI, roomserverAliasAPI, workerStates, + rsAPI, workerStates, ) if err := consumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start appservice roomserver consumer") @@ -100,7 +99,7 @@ func SetupAppServiceAPIComponent( // Set up HTTP Endpoints routing.Setup( - base.APIMux, base.Cfg, roomserverQueryAPI, roomserverAliasAPI, + base.APIMux, base.Cfg, rsAPI, accountsDB, federation, transactionsCache, ) diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index 3bd364c58..b7f689249 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -35,8 +35,7 @@ type OutputRoomEventConsumer struct { roomServerConsumer *common.ContinualConsumer db accounts.Database asDB storage.Database - query api.RoomserverQueryAPI - alias api.RoomserverAliasAPI + rsAPI api.RoomserverInternalAPI serverName string workerStates []types.ApplicationServiceWorkerState } @@ -48,8 +47,7 @@ func NewOutputRoomEventConsumer( kafkaConsumer sarama.Consumer, store accounts.Database, appserviceDB storage.Database, - queryAPI api.RoomserverQueryAPI, - aliasAPI api.RoomserverAliasAPI, + rsAPI api.RoomserverInternalAPI, workerStates []types.ApplicationServiceWorkerState, ) *OutputRoomEventConsumer { consumer := common.ContinualConsumer{ @@ -61,8 +59,7 @@ func NewOutputRoomEventConsumer( roomServerConsumer: &consumer, db: store, asDB: appserviceDB, - query: queryAPI, - alias: aliasAPI, + rsAPI: rsAPI, serverName: string(cfg.Matrix.ServerName), workerStates: workerStates, } @@ -139,7 +136,7 @@ func (s *OutputRoomEventConsumer) lookupMissingStateEvents( // Request the missing events from the roomserver eventReq := api.QueryEventsByIDRequest{EventIDs: missing} var eventResp api.QueryEventsByIDResponse - if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { + if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { return nil, err } @@ -200,7 +197,7 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont // Check all known room aliases of the room the event came from queryReq := api.GetAliasesForRoomIDRequest{RoomID: event.RoomID()} var queryRes api.GetAliasesForRoomIDResponse - if err := s.alias.GetAliasesForRoomID(ctx, &queryReq, &queryRes); err == nil { + if err := s.rsAPI.GetAliasesForRoomID(ctx, &queryReq, &queryRes); err == nil { for _, alias := range queryRes.Aliases { if appservice.IsInterestedInRoomAlias(alias) { return true diff --git a/appservice/routing/routing.go b/appservice/routing/routing.go index 42fa80520..9f59e05f7 100644 --- a/appservice/routing/routing.go +++ b/appservice/routing/routing.go @@ -37,7 +37,7 @@ const pathPrefixApp = "/_matrix/app/v1" // nolint: gocyclo func Setup( apiMux *mux.Router, cfg *config.Dendrite, // nolint: unparam - queryAPI api.RoomserverQueryAPI, aliasAPI api.RoomserverAliasAPI, // nolint: unparam + rsAPI api.RoomserverInternalAPI, // nolint: unparam accountDB accounts.Database, // nolint: unparam federation *gomatrixserverlib.FederationClient, // nolint: unparam transactionsCache *transactions.Cache, // nolint: unparam diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 6a857e52c..f81e0242c 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -38,15 +38,13 @@ func SetupClientAPIComponent( accountsDB accounts.Database, federation *gomatrixserverlib.FederationClient, keyRing *gomatrixserverlib.KeyRing, - aliasAPI roomserverAPI.RoomserverAliasAPI, - inputAPI roomserverAPI.RoomserverInputAPI, - queryAPI roomserverAPI.RoomserverQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, eduInputAPI eduServerAPI.EDUServerInputAPI, asAPI appserviceAPI.AppServiceQueryAPI, transactionsCache *transactions.Cache, fsAPI federationSenderAPI.FederationSenderInternalAPI, ) { - roomserverProducer := producers.NewRoomserverProducer(inputAPI, queryAPI) + roomserverProducer := producers.NewRoomserverProducer(rsAPI) eduProducer := producers.NewEDUServerProducer(eduInputAPI) userUpdateProducer := &producers.UserUpdateProducer{ @@ -60,14 +58,14 @@ func SetupClientAPIComponent( } consumer := consumers.NewOutputRoomEventConsumer( - base.Cfg, base.KafkaConsumer, accountsDB, queryAPI, + base.Cfg, base.KafkaConsumer, accountsDB, rsAPI, ) if err := consumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start room server consumer") } routing.Setup( - base.APIMux, base.Cfg, roomserverProducer, queryAPI, aliasAPI, asAPI, + base.APIMux, base.Cfg, roomserverProducer, rsAPI, asAPI, accountsDB, deviceDB, federation, *keyRing, userUpdateProducer, syncProducer, eduProducer, transactionsCache, fsAPI, ) diff --git a/clientapi/consumers/roomserver.go b/clientapi/consumers/roomserver.go index 3c7905721..d0c91e88f 100644 --- a/clientapi/consumers/roomserver.go +++ b/clientapi/consumers/roomserver.go @@ -30,10 +30,10 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { - roomServerConsumer *common.ContinualConsumer - db accounts.Database - query api.RoomserverQueryAPI - serverName string + rsAPI api.RoomserverInternalAPI + rsConsumer *common.ContinualConsumer + db accounts.Database + serverName string } // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. @@ -41,7 +41,7 @@ func NewOutputRoomEventConsumer( cfg *config.Dendrite, kafkaConsumer sarama.Consumer, store accounts.Database, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, ) *OutputRoomEventConsumer { consumer := common.ContinualConsumer{ @@ -50,10 +50,10 @@ func NewOutputRoomEventConsumer( PartitionStore: store, } s := &OutputRoomEventConsumer{ - roomServerConsumer: &consumer, - db: store, - query: queryAPI, - serverName: string(cfg.Matrix.ServerName), + rsConsumer: &consumer, + db: store, + rsAPI: rsAPI, + serverName: string(cfg.Matrix.ServerName), } consumer.ProcessMessage = s.onMessage @@ -62,7 +62,7 @@ func NewOutputRoomEventConsumer( // Start consuming from room servers func (s *OutputRoomEventConsumer) Start() error { - return s.roomServerConsumer.Start() + return s.rsConsumer.Start() } // onMessage is called when the sync server receives a new event from the room server output log. @@ -134,7 +134,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( // Request the missing events from the roomserver eventReq := api.QueryEventsByIDRequest{EventIDs: missing} var eventResp api.QueryEventsByIDResponse - if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { + if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { return nil, err } diff --git a/clientapi/producers/roomserver.go b/clientapi/producers/roomserver.go index fac1e3c7c..a804abfe9 100644 --- a/clientapi/producers/roomserver.go +++ b/clientapi/producers/roomserver.go @@ -23,15 +23,13 @@ import ( // RoomserverProducer produces events for the roomserver to consume. type RoomserverProducer struct { - InputAPI api.RoomserverInputAPI - QueryAPI api.RoomserverQueryAPI + RsAPI api.RoomserverInternalAPI } // NewRoomserverProducer creates a new RoomserverProducer -func NewRoomserverProducer(inputAPI api.RoomserverInputAPI, queryAPI api.RoomserverQueryAPI) *RoomserverProducer { +func NewRoomserverProducer(rsAPI api.RoomserverInternalAPI) *RoomserverProducer { return &RoomserverProducer{ - InputAPI: inputAPI, - QueryAPI: queryAPI, + RsAPI: rsAPI, } } @@ -95,7 +93,7 @@ func (c *RoomserverProducer) SendInputRoomEvents( ) (eventID string, err error) { request := api.InputRoomEventsRequest{InputRoomEvents: ires} var response api.InputRoomEventsResponse - err = c.InputAPI.InputRoomEvents(ctx, &request, &response) + err = c.RsAPI.InputRoomEvents(ctx, &request, &response) eventID = response.EventID return } @@ -118,5 +116,5 @@ func (c *RoomserverProducer) SendInvite( }}, } var response api.InputRoomEventsResponse - return c.InputAPI.InputRoomEvents(ctx, &request, &response) + return c.RsAPI.InputRoomEvents(ctx, &request, &response) } diff --git a/clientapi/routing/capabilities.go b/clientapi/routing/capabilities.go index 1792c6308..199b15240 100644 --- a/clientapi/routing/capabilities.go +++ b/clientapi/routing/capabilities.go @@ -26,11 +26,11 @@ import ( // SendMembership implements PUT /rooms/{roomID}/(join|kick|ban|unban|leave|invite) // by building a m.room.member event then sending it to the room server func GetCapabilities( - req *http.Request, queryAPI roomserverAPI.RoomserverQueryAPI, + req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, ) util.JSONResponse { roomVersionsQueryReq := roomserverAPI.QueryRoomVersionCapabilitiesRequest{} roomVersionsQueryRes := roomserverAPI.QueryRoomVersionCapabilitiesResponse{} - if err := queryAPI.QueryRoomVersionCapabilities( + if err := rsAPI.QueryRoomVersionCapabilities( req.Context(), &roomVersionsQueryReq, &roomVersionsQueryRes, diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index ef11e8b3e..28e2b1514 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -137,13 +137,13 @@ type fledglingEvent struct { func CreateRoom( req *http.Request, device *authtypes.Device, cfg *config.Dendrite, producer *producers.RoomserverProducer, - accountDB accounts.Database, aliasAPI roomserverAPI.RoomserverAliasAPI, + accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { // TODO (#267): Check room ID doesn't clash with an existing one, and we // probably shouldn't be using pseudo-random strings, maybe GUIDs? roomID := fmt.Sprintf("!%s:%s", util.RandomString(16), cfg.Matrix.ServerName) - return createRoom(req, device, cfg, roomID, producer, accountDB, aliasAPI, asAPI) + return createRoom(req, device, cfg, roomID, producer, accountDB, rsAPI, asAPI) } // createRoom implements /createRoom @@ -151,7 +151,7 @@ func CreateRoom( func createRoom( req *http.Request, device *authtypes.Device, cfg *config.Dendrite, roomID string, producer *producers.RoomserverProducer, - accountDB accounts.Database, aliasAPI roomserverAPI.RoomserverAliasAPI, + accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { logger := util.GetLogger(req.Context()) @@ -340,7 +340,7 @@ func createRoom( } var aliasResp roomserverAPI.SetRoomAliasResponse - err = aliasAPI.SetRoomAlias(req.Context(), &aliasReq, &aliasResp) + err = rsAPI.SetRoomAlias(req.Context(), &aliasReq, &aliasResp) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.SetRoomAlias failed") return jsonerror.InternalServerError() diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index 101ba11ff..a0a60a47a 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -46,7 +46,7 @@ func DirectoryRoom( roomAlias string, federation *gomatrixserverlib.FederationClient, cfg *config.Dendrite, - rsAPI roomserverAPI.RoomserverAliasAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, fedSenderAPI federationSenderAPI.FederationSenderInternalAPI, ) util.JSONResponse { _, domain, err := gomatrixserverlib.SplitID('#', roomAlias) @@ -115,7 +115,7 @@ func SetLocalAlias( device *authtypes.Device, alias string, cfg *config.Dendrite, - aliasAPI roomserverAPI.RoomserverAliasAPI, + aliasAPI roomserverAPI.RoomserverInternalAPI, ) util.JSONResponse { _, domain, err := gomatrixserverlib.SplitID('#', alias) if err != nil { @@ -190,7 +190,7 @@ func RemoveLocalAlias( req *http.Request, device *authtypes.Device, alias string, - aliasAPI roomserverAPI.RoomserverAliasAPI, + aliasAPI roomserverAPI.RoomserverInternalAPI, ) util.JSONResponse { creatorQueryReq := roomserverAPI.GetCreatorIDForAliasRequest{ diff --git a/clientapi/routing/getevent.go b/clientapi/routing/getevent.go index 2d3152510..bf49968de 100644 --- a/clientapi/routing/getevent.go +++ b/clientapi/routing/getevent.go @@ -44,7 +44,7 @@ func GetEvent( roomID string, eventID string, cfg *config.Dendrite, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, federation *gomatrixserverlib.FederationClient, keyRing gomatrixserverlib.KeyRing, ) util.JSONResponse { @@ -52,7 +52,7 @@ func GetEvent( EventIDs: []string{eventID}, } var eventsResp api.QueryEventsByIDResponse - err := queryAPI.QueryEventsByID(req.Context(), &eventsReq, &eventsResp) + err := rsAPI.QueryEventsByID(req.Context(), &eventsReq, &eventsResp) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryEventsByID failed") return jsonerror.InternalServerError() @@ -88,7 +88,7 @@ func GetEvent( }}, } var stateResp api.QueryStateAfterEventsResponse - if err := queryAPI.QueryStateAfterEvents(req.Context(), &stateReq, &stateResp); err != nil { + if err := rsAPI.QueryStateAfterEvents(req.Context(), &stateReq, &stateResp); err != nil { util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryStateAfterEvents failed") return jsonerror.InternalServerError() } diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index f55d1b6ab..df83c2a9f 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -43,8 +43,7 @@ func JoinRoomByIDOrAlias( cfg *config.Dendrite, federation *gomatrixserverlib.FederationClient, producer *producers.RoomserverProducer, - queryAPI roomserverAPI.RoomserverQueryAPI, - aliasAPI roomserverAPI.RoomserverAliasAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, fsAPI federationSenderAPI.FederationSenderInternalAPI, keyRing gomatrixserverlib.KeyRing, accountDB accounts.Database, @@ -80,7 +79,7 @@ func JoinRoomByIDOrAlias( r := joinRoomReq{ req, evTime, content, device.UserID, cfg, federation, producer, - queryAPI, aliasAPI, fsAPI, keyRing, + rsAPI, fsAPI, keyRing, } if strings.HasPrefix(roomIDOrAlias, "!") { @@ -106,8 +105,7 @@ type joinRoomReq struct { cfg *config.Dendrite federation *gomatrixserverlib.FederationClient producer *producers.RoomserverProducer - queryAPI roomserverAPI.RoomserverQueryAPI - aliasAPI roomserverAPI.RoomserverAliasAPI + rsAPI roomserverAPI.RoomserverInternalAPI fsAPI federationSenderAPI.FederationSenderInternalAPI keyRing gomatrixserverlib.KeyRing } @@ -124,7 +122,7 @@ func (r joinRoomReq) joinRoomByID(roomID string) util.JSONResponse { RoomID: roomID, TargetUserID: r.userID, } var queryRes roomserverAPI.QueryInvitesForUserResponse - if err := r.queryAPI.QueryInvitesForUser(r.req.Context(), &queryReq, &queryRes); err != nil { + if err := r.rsAPI.QueryInvitesForUser(r.req.Context(), &queryReq, &queryRes); err != nil { util.GetLogger(r.req.Context()).WithError(err).Error("r.queryAPI.QueryInvitesForUser failed") return jsonerror.InternalServerError() } @@ -172,7 +170,7 @@ func (r joinRoomReq) joinRoomByAlias(roomAlias string) util.JSONResponse { if domain == r.cfg.Matrix.ServerName { queryReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: roomAlias} var queryRes roomserverAPI.GetRoomIDForAliasResponse - if err = r.aliasAPI.GetRoomIDForAlias(r.req.Context(), &queryReq, &queryRes); err != nil { + if err = r.rsAPI.GetRoomIDForAlias(r.req.Context(), &queryReq, &queryRes); err != nil { util.GetLogger(r.req.Context()).WithError(err).Error("r.aliasAPI.GetRoomIDForAlias failed") return jsonerror.InternalServerError() } @@ -243,7 +241,7 @@ func (r joinRoomReq) joinRoomUsingServers( } queryRes := roomserverAPI.QueryLatestEventsAndStateResponse{} - event, err := common.BuildEvent(r.req.Context(), &eb, r.cfg, r.evTime, r.queryAPI, &queryRes) + event, err := common.BuildEvent(r.req.Context(), &eb, r.cfg, r.evTime, r.rsAPI, &queryRes) if err == nil { // If we have successfully built an event at this point then we can // assert that the room is a local room, as BuildEvent was able to diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index dff194dd3..9030f9f7e 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -45,12 +45,12 @@ var errMissingUserID = errors.New("'user_id' must be supplied") func SendMembership( req *http.Request, accountDB accounts.Database, device *authtypes.Device, roomID string, membership string, cfg *config.Dendrite, - queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, producer *producers.RoomserverProducer, ) util.JSONResponse { verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := queryAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { + if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UnsupportedRoomVersion(err.Error()), @@ -71,7 +71,7 @@ func SendMembership( } inviteStored, jsonErrResp := checkAndProcessThreepid( - req, device, &body, cfg, queryAPI, accountDB, producer, + req, device, &body, cfg, rsAPI, accountDB, producer, membership, roomID, evTime, ) if jsonErrResp != nil { @@ -89,7 +89,7 @@ func SendMembership( } event, err := buildMembershipEvent( - req.Context(), body, accountDB, device, membership, roomID, cfg, evTime, queryAPI, asAPI, + req.Context(), body, accountDB, device, membership, roomID, cfg, evTime, rsAPI, asAPI, ) if err == errMissingUserID { return util.JSONResponse{ @@ -153,7 +153,7 @@ func buildMembershipEvent( device *authtypes.Device, membership, roomID string, cfg *config.Dendrite, evTime time.Time, - queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) (*gomatrixserverlib.Event, error) { stateKey, reason, err := getMembershipStateKey(body, device, membership) if err != nil { @@ -188,7 +188,7 @@ func buildMembershipEvent( return nil, err } - return common.BuildEvent(ctx, &builder, cfg, evTime, queryAPI, nil) + return common.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil) } // loadProfile lookups the profile of a given user from the database and returns @@ -248,7 +248,7 @@ func checkAndProcessThreepid( device *authtypes.Device, body *threepid.MembershipRequest, cfg *config.Dendrite, - queryAPI roomserverAPI.RoomserverQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, accountDB accounts.Database, producer *producers.RoomserverProducer, membership, roomID string, @@ -256,7 +256,7 @@ func checkAndProcessThreepid( ) (inviteStored bool, errRes *util.JSONResponse) { inviteStored, err := threepid.CheckAndProcessInvite( - req.Context(), device, body, cfg, queryAPI, accountDB, producer, + req.Context(), device, body, cfg, rsAPI, accountDB, producer, membership, roomID, evTime, ) if err == threepid.ErrMissingParameter { diff --git a/clientapi/routing/memberships.go b/clientapi/routing/memberships.go index 0b846e5e3..f5d9bc4c5 100644 --- a/clientapi/routing/memberships.go +++ b/clientapi/routing/memberships.go @@ -39,7 +39,7 @@ type getJoinedRoomsResponse struct { func GetMemberships( req *http.Request, device *authtypes.Device, roomID string, joinedOnly bool, _ *config.Dendrite, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { queryReq := api.QueryMembershipsForRoomRequest{ JoinedOnly: joinedOnly, @@ -47,8 +47,8 @@ func GetMemberships( Sender: device.UserID, } var queryRes api.QueryMembershipsForRoomResponse - if err := queryAPI.QueryMembershipsForRoom(req.Context(), &queryReq, &queryRes); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryMembershipsForRoom failed") + if err := rsAPI.QueryMembershipsForRoom(req.Context(), &queryReq, &queryRes); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryMembershipsForRoom failed") return jsonerror.InternalServerError() } diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index a51c55ea5..b51533e45 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -94,7 +94,7 @@ func GetAvatarURL( func SetAvatarURL( req *http.Request, accountDB accounts.Database, device *authtypes.Device, userID string, producer *producers.UserUpdateProducer, cfg *config.Dendrite, - rsProducer *producers.RoomserverProducer, queryAPI api.RoomserverQueryAPI, + rsProducer *producers.RoomserverProducer, rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { if userID != device.UserID { return util.JSONResponse{ @@ -154,7 +154,7 @@ func SetAvatarURL( } events, err := buildMembershipEvents( - req.Context(), memberships, newProfile, userID, cfg, evTime, queryAPI, + req.Context(), memberships, newProfile, userID, cfg, evTime, rsAPI, ) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvents failed") @@ -208,7 +208,7 @@ func GetDisplayName( func SetDisplayName( req *http.Request, accountDB accounts.Database, device *authtypes.Device, userID string, producer *producers.UserUpdateProducer, cfg *config.Dendrite, - rsProducer *producers.RoomserverProducer, queryAPI api.RoomserverQueryAPI, + rsProducer *producers.RoomserverProducer, rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { if userID != device.UserID { return util.JSONResponse{ @@ -268,7 +268,7 @@ func SetDisplayName( } events, err := buildMembershipEvents( - req.Context(), memberships, newProfile, userID, cfg, evTime, queryAPI, + req.Context(), memberships, newProfile, userID, cfg, evTime, rsAPI, ) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvents failed") @@ -337,14 +337,14 @@ func buildMembershipEvents( ctx context.Context, memberships []authtypes.Membership, newProfile authtypes.Profile, userID string, cfg *config.Dendrite, - evTime time.Time, queryAPI api.RoomserverQueryAPI, + evTime time.Time, rsAPI api.RoomserverInternalAPI, ) ([]gomatrixserverlib.HeaderedEvent, error) { evs := []gomatrixserverlib.HeaderedEvent{} for _, membership := range memberships { verReq := api.QueryRoomVersionForRoomRequest{RoomID: membership.RoomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := queryAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil { + if err := rsAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil { return []gomatrixserverlib.HeaderedEvent{}, err } @@ -366,7 +366,7 @@ func buildMembershipEvents( return nil, err } - event, err := common.BuildEvent(ctx, &builder, cfg, evTime, queryAPI, nil) + event, err := common.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil) if err != nil { return nil, err } diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index e62b51935..42b391de6 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -49,8 +49,7 @@ const pathPrefixUnstable = "/_matrix/client/unstable" func Setup( apiMux *mux.Router, cfg *config.Dendrite, producer *producers.RoomserverProducer, - queryAPI roomserverAPI.RoomserverQueryAPI, - aliasAPI roomserverAPI.RoomserverAliasAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, accountDB accounts.Database, deviceDB devices.Database, @@ -91,7 +90,7 @@ func Setup( r0mux.Handle("/createRoom", common.MakeAuthAPI("createRoom", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - return CreateRoom(req, device, cfg, producer, accountDB, aliasAPI, asAPI) + return CreateRoom(req, device, cfg, producer, accountDB, rsAPI, asAPI) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/join/{roomIDOrAlias}", @@ -102,7 +101,7 @@ func Setup( } return JoinRoomByIDOrAlias( req, device, vars["roomIDOrAlias"], cfg, federation, producer, - queryAPI, aliasAPI, federationSender, keyRing, accountDB, + rsAPI, federationSender, keyRing, accountDB, ) }), ).Methods(http.MethodPost, http.MethodOptions) @@ -118,7 +117,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, queryAPI, asAPI, producer) + return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, rsAPI, asAPI, producer) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/send/{eventType}", @@ -127,7 +126,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, queryAPI, producer, nil) + return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, rsAPI, producer, nil) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}", @@ -138,7 +137,7 @@ func Setup( } txnID := vars["txnID"] return SendEvent(req, device, vars["roomID"], vars["eventType"], &txnID, - nil, cfg, queryAPI, producer, transactionsCache) + nil, cfg, rsAPI, producer, transactionsCache) }), ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/event/{eventID}", @@ -147,7 +146,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, queryAPI, federation, keyRing) + return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI, federation, keyRing) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -156,7 +155,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return OnIncomingStateRequest(req.Context(), queryAPI, vars["roomID"]) + return OnIncomingStateRequest(req.Context(), rsAPI, vars["roomID"]) })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{type}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { @@ -164,7 +163,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return OnIncomingStateTypeRequest(req.Context(), queryAPI, vars["roomID"], vars["type"], "") + return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], "") })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { @@ -172,7 +171,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return OnIncomingStateTypeRequest(req.Context(), queryAPI, vars["roomID"], vars["type"], vars["stateKey"]) + return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], vars["stateKey"]) })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}", @@ -187,7 +186,7 @@ func Setup( if strings.HasSuffix(eventType, "/") { eventType = eventType[:len(eventType)-1] } - return SendEvent(req, device, vars["roomID"], eventType, nil, &emptyString, cfg, queryAPI, producer, nil) + return SendEvent(req, device, vars["roomID"], eventType, nil, &emptyString, cfg, rsAPI, producer, nil) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -198,7 +197,7 @@ func Setup( return util.ErrorResponse(err) } stateKey := vars["stateKey"] - return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, &stateKey, cfg, queryAPI, producer, nil) + return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, &stateKey, cfg, rsAPI, producer, nil) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -220,7 +219,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return DirectoryRoom(req, vars["roomAlias"], federation, cfg, aliasAPI, federationSender) + return DirectoryRoom(req, vars["roomAlias"], federation, cfg, rsAPI, federationSender) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -230,7 +229,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SetLocalAlias(req, device, vars["roomAlias"], cfg, aliasAPI) + return SetLocalAlias(req, device, vars["roomAlias"], cfg, rsAPI) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -240,7 +239,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return RemoveLocalAlias(req, device, vars["roomAlias"], aliasAPI) + return RemoveLocalAlias(req, device, vars["roomAlias"], rsAPI) }), ).Methods(http.MethodDelete, http.MethodOptions) @@ -354,7 +353,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SetAvatarURL(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, queryAPI) + return SetAvatarURL(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, rsAPI) }), ).Methods(http.MethodPut, http.MethodOptions) // Browsers use the OPTIONS HTTP method to check if the CORS policy allows @@ -376,7 +375,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SetDisplayName(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, queryAPI) + return SetDisplayName(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, rsAPI) }), ).Methods(http.MethodPut, http.MethodOptions) // Browsers use the OPTIONS HTTP method to check if the CORS policy allows @@ -489,7 +488,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetMemberships(req, device, vars["roomID"], false, cfg, queryAPI) + return GetMemberships(req, device, vars["roomID"], false, cfg, rsAPI) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -499,7 +498,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetMemberships(req, device, vars["roomID"], true, cfg, queryAPI) + return GetMemberships(req, device, vars["roomID"], true, cfg, rsAPI) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -603,7 +602,7 @@ func Setup( r0mux.Handle("/capabilities", common.MakeAuthAPI("capabilities", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - return GetCapabilities(req, queryAPI) + return GetCapabilities(req, rsAPI) }), ).Methods(http.MethodGet) } diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index 5b2cd8ad4..7280dcd94 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -45,13 +45,13 @@ func SendEvent( device *authtypes.Device, roomID, eventType string, txnID, stateKey *string, cfg *config.Dendrite, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, producer *producers.RoomserverProducer, txnCache *transactions.Cache, ) util.JSONResponse { verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := queryAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { + if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UnsupportedRoomVersion(err.Error()), @@ -65,7 +65,7 @@ func SendEvent( } } - e, resErr := generateSendEvent(req, device, roomID, eventType, stateKey, cfg, queryAPI) + e, resErr := generateSendEvent(req, device, roomID, eventType, stateKey, cfg, rsAPI) if resErr != nil { return *resErr } @@ -115,7 +115,7 @@ func generateSendEvent( device *authtypes.Device, roomID, eventType string, stateKey *string, cfg *config.Dendrite, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, ) (*gomatrixserverlib.Event, *util.JSONResponse) { // parse the incoming http request userID := device.UserID @@ -148,7 +148,7 @@ func generateSendEvent( } var queryRes api.QueryLatestEventsAndStateResponse - e, err := common.BuildEvent(req.Context(), &builder, cfg, evTime, queryAPI, &queryRes) + e, err := common.BuildEvent(req.Context(), &builder, cfg, evTime, rsAPI, &queryRes) if err == common.ErrRoomNoExists { return nil, &util.JSONResponse{ Code: http.StatusNotFound, diff --git a/clientapi/routing/state.go b/clientapi/routing/state.go index c243eec0f..e3e5bdb6d 100644 --- a/clientapi/routing/state.go +++ b/clientapi/routing/state.go @@ -40,7 +40,7 @@ type stateEventInStateResp struct { // TODO: Check if the user is in the room. If not, check if the room's history // is publicly visible. Current behaviour is returning an empty array if the // user cannot see the room's history. -func OnIncomingStateRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI, roomID string) util.JSONResponse { +func OnIncomingStateRequest(ctx context.Context, rsAPI api.RoomserverInternalAPI, roomID string) util.JSONResponse { // TODO(#287): Auth request and handle the case where the user has left (where // we should return the state at the poin they left) stateReq := api.QueryLatestEventsAndStateRequest{ @@ -48,7 +48,7 @@ func OnIncomingStateRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI } stateRes := api.QueryLatestEventsAndStateResponse{} - if err := queryAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil { + if err := rsAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil { util.GetLogger(ctx).WithError(err).Error("queryAPI.QueryLatestEventsAndState failed") return jsonerror.InternalServerError() } @@ -98,7 +98,7 @@ func OnIncomingStateRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI // /rooms/{roomID}/state/{type}/{statekey} request. It will look in current // state to see if there is an event with that type and state key, if there // is then (by default) we return the content, otherwise a 404. -func OnIncomingStateTypeRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI, roomID string, evType, stateKey string) util.JSONResponse { +func OnIncomingStateTypeRequest(ctx context.Context, rsAPI api.RoomserverInternalAPI, roomID string, evType, stateKey string) util.JSONResponse { // TODO(#287): Auth request and handle the case where the user has left (where // we should return the state at the poin they left) util.GetLogger(ctx).WithFields(log.Fields{ @@ -118,7 +118,7 @@ func OnIncomingStateTypeRequest(ctx context.Context, queryAPI api.RoomserverQuer } stateRes := api.QueryLatestEventsAndStateResponse{} - if err := queryAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil { + if err := rsAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil { util.GetLogger(ctx).WithError(err).Error("queryAPI.QueryLatestEventsAndState failed") return jsonerror.InternalServerError() } diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index e34e91b56..5e7e4f2b4 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -87,7 +87,7 @@ var ( func CheckAndProcessInvite( ctx context.Context, device *authtypes.Device, body *MembershipRequest, cfg *config.Dendrite, - queryAPI api.RoomserverQueryAPI, db accounts.Database, + rsAPI api.RoomserverInternalAPI, db accounts.Database, producer *producers.RoomserverProducer, membership string, roomID string, evTime time.Time, ) (inviteStoredOnIDServer bool, err error) { @@ -112,7 +112,7 @@ func CheckAndProcessInvite( // "m.room.third_party_invite" have to be emitted from the data in // storeInviteRes. err = emit3PIDInviteEvent( - ctx, body, storeInviteRes, device, roomID, cfg, queryAPI, producer, evTime, + ctx, body, storeInviteRes, device, roomID, cfg, rsAPI, producer, evTime, ) inviteStoredOnIDServer = err == nil @@ -331,7 +331,7 @@ func emit3PIDInviteEvent( ctx context.Context, body *MembershipRequest, res *idServerStoreInviteResponse, device *authtypes.Device, roomID string, cfg *config.Dendrite, - queryAPI api.RoomserverQueryAPI, producer *producers.RoomserverProducer, + rsAPI api.RoomserverInternalAPI, producer *producers.RoomserverProducer, evTime time.Time, ) error { builder := &gomatrixserverlib.EventBuilder{ @@ -354,7 +354,7 @@ func emit3PIDInviteEvent( } queryRes := api.QueryLatestEventsAndStateResponse{} - event, err := common.BuildEvent(ctx, builder, cfg, evTime, queryAPI, &queryRes) + event, err := common.BuildEvent(ctx, builder, cfg, evTime, rsAPI, &queryRes) if err != nil { return err } diff --git a/cmd/dendrite-appservice-server/main.go b/cmd/dendrite-appservice-server/main.go index f203969f4..91ca707e7 100644 --- a/cmd/dendrite-appservice-server/main.go +++ b/cmd/dendrite-appservice-server/main.go @@ -28,11 +28,11 @@ func main() { accountDB := base.CreateAccountsDB() deviceDB := base.CreateDeviceDB() federation := base.CreateFederationClient() - alias, _, query := base.CreateHTTPRoomserverAPIs() + rsAPI := base.CreateHTTPRoomserverAPIs() cache := transactions.New() appservice.SetupAppServiceAPIComponent( - base, accountDB, deviceDB, federation, alias, query, cache, + base, accountDB, deviceDB, federation, rsAPI, cache, ) base.SetupAndServeHTTP(string(base.Cfg.Bind.AppServiceAPI), string(base.Cfg.Listen.AppServiceAPI)) diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index c8f629689..1ce66db2c 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -36,13 +36,14 @@ func main() { keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) asQuery := base.CreateHTTPAppServiceAPIs() - alias, input, query := base.CreateHTTPRoomserverAPIs() + rsAPI := base.CreateHTTPRoomserverAPIs() fsAPI := base.CreateHTTPFederationSenderAPIs() + rsAPI.SetFederationSenderAPI(fsAPI) eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, federation, &keyRing, - alias, input, query, eduInputAPI, asQuery, transactions.New(), fsAPI, + rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, ) base.SetupAndServeHTTP(string(base.Cfg.Bind.ClientAPI), string(base.Cfg.Listen.ClientAPI)) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index a2a4675b3..28c7153fd 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -148,27 +148,34 @@ func main() { federation := createFederationClient(base) keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - alias, input, query := roomserver.SetupRoomServerComponent(&base.Base, keyRing, federation) - eduInputAPI := eduserver.SetupEDUServerComponent(&base.Base, cache.New()) - asQuery := appservice.SetupAppServiceAPIComponent( - &base.Base, accountDB, deviceDB, federation, alias, query, transactions.New(), + rsAPI := roomserver.SetupRoomServerComponent( + &base.Base, keyRing, federation, ) - fsAPI := federationsender.SetupFederationSenderComponent(&base.Base, federation, query, input, &keyRing) + eduInputAPI := eduserver.SetupEDUServerComponent( + &base.Base, cache.New(), + ) + asAPI := appservice.SetupAppServiceAPIComponent( + &base.Base, accountDB, deviceDB, federation, rsAPI, transactions.New(), + ) + fsAPI := federationsender.SetupFederationSenderComponent( + &base.Base, federation, rsAPI, &keyRing, + ) + rsAPI.SetFederationSenderAPI(fsAPI) clientapi.SetupClientAPIComponent( &base.Base, deviceDB, accountDB, - federation, &keyRing, alias, input, query, - eduInputAPI, asQuery, transactions.New(), fsAPI, + federation, &keyRing, rsAPI, + eduInputAPI, asAPI, transactions.New(), fsAPI, ) eduProducer := producers.NewEDUServerProducer(eduInputAPI) - federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fsAPI, eduProducer) + federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, &keyRing, rsAPI, asAPI, fsAPI, eduProducer) mediaapi.SetupMediaAPIComponent(&base.Base, deviceDB) publicRoomsDB, err := storage.NewPublicRoomsServerDatabaseWithPubSub(string(base.Base.Cfg.Database.PublicRoomsAPI), base.LibP2PPubsub) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - publicroomsapi.SetupPublicRoomsAPIComponent(&base.Base, deviceDB, publicRoomsDB, query, federation, nil) // Check this later - syncapi.SetupSyncAPIComponent(&base.Base, deviceDB, accountDB, query, federation, &cfg) + publicroomsapi.SetupPublicRoomsAPIComponent(&base.Base, deviceDB, publicRoomsDB, rsAPI, federation, nil) // Check this later + syncapi.SetupSyncAPIComponent(&base.Base, deviceDB, accountDB, rsAPI, federation, &cfg) httpHandler := common.WrapHandlerInCORS(base.Base.APIMux) diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index 4267cf166..d829326a7 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -35,14 +35,15 @@ func main() { fsAPI := base.CreateHTTPFederationSenderAPIs() keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - alias, input, query := base.CreateHTTPRoomserverAPIs() - asQuery := base.CreateHTTPAppServiceAPIs() + rsAPI := base.CreateHTTPRoomserverAPIs() + asAPI := base.CreateHTTPAppServiceAPIs() + rsAPI.SetFederationSenderAPI(fsAPI) eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) eduProducer := producers.NewEDUServerProducer(eduInputAPI) federationapi.SetupFederationAPIComponent( base, accountDB, deviceDB, federation, &keyRing, - alias, input, query, asQuery, fsAPI, eduProducer, + rsAPI, asAPI, fsAPI, eduProducer, ) base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI)) diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-federation-sender-server/main.go index f8d43b990..0daac1bcb 100644 --- a/cmd/dendrite-federation-sender-server/main.go +++ b/cmd/dendrite-federation-sender-server/main.go @@ -28,12 +28,11 @@ func main() { federation := base.CreateFederationClient() keyDB := base.CreateKeyDB() keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - - _, input, query := base.CreateHTTPRoomserverAPIs() - - federationsender.SetupFederationSenderComponent( - base, federation, query, input, &keyRing, + rsAPI := base.CreateHTTPRoomserverAPIs() + fsAPI := federationsender.SetupFederationSenderComponent( + base, federation, rsAPI, &keyRing, ) + rsAPI.SetFederationSenderAPI(fsAPI) base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationSender), string(base.Cfg.Listen.FederationSender)) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index f43f8b04c..060019711 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -57,28 +57,34 @@ func main() { federation := base.CreateFederationClient() keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - alias, input, query := roomserver.SetupRoomServerComponent(base, keyRing, federation) - eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) - asQuery := appservice.SetupAppServiceAPIComponent( - base, accountDB, deviceDB, federation, alias, query, transactions.New(), + rsAPI := roomserver.SetupRoomServerComponent( + base, keyRing, federation, ) - fsAPI := federationsender.SetupFederationSenderComponent(base, federation, query, input, &keyRing) - input.SetFederationSenderAPI(fsAPI) + eduInputAPI := eduserver.SetupEDUServerComponent( + base, cache.New(), + ) + asAPI := appservice.SetupAppServiceAPIComponent( + base, accountDB, deviceDB, federation, rsAPI, transactions.New(), + ) + fsAPI := federationsender.SetupFederationSenderComponent( + base, federation, rsAPI, &keyRing, + ) + rsAPI.SetFederationSenderAPI(fsAPI) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, - federation, &keyRing, alias, input, query, - eduInputAPI, asQuery, transactions.New(), fsAPI, + federation, &keyRing, rsAPI, + eduInputAPI, asAPI, transactions.New(), fsAPI, ) eduProducer := producers.NewEDUServerProducer(eduInputAPI) - federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fsAPI, eduProducer) + federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asAPI, fsAPI, eduProducer) mediaapi.SetupMediaAPIComponent(base, deviceDB) publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI)) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, query, federation, nil) - syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg) + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, federation, nil) + syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg) httpHandler := common.WrapHandlerInCORS(base.APIMux) diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go index f6a782f66..fca39a2fe 100644 --- a/cmd/dendrite-public-rooms-api-server/main.go +++ b/cmd/dendrite-public-rooms-api-server/main.go @@ -28,12 +28,15 @@ func main() { deviceDB := base.CreateDeviceDB() - _, _, query := base.CreateHTTPRoomserverAPIs() + fsAPI := base.CreateHTTPFederationSenderAPIs() + rsAPI := base.CreateHTTPRoomserverAPIs() + rsAPI.SetFederationSenderAPI(fsAPI) + publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI)) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, query, nil, nil) + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, nil, nil) base.SetupAndServeHTTP(string(base.Cfg.Bind.PublicRoomsAPI), string(base.Cfg.Listen.PublicRoomsAPI)) diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go index 3f9913e24..41149ad90 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-room-server/main.go @@ -31,8 +31,8 @@ func main() { keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) fsAPI := base.CreateHTTPFederationSenderAPIs() - _, input, _ := roomserver.SetupRoomServerComponent(base, keyRing, federation) - input.SetFederationSenderAPI(fsAPI) + rsAPI := roomserver.SetupRoomServerComponent(base, keyRing, federation) + rsAPI.SetFederationSenderAPI(fsAPI) base.SetupAndServeHTTP(string(base.Cfg.Bind.RoomServer), string(base.Cfg.Listen.RoomServer)) diff --git a/cmd/dendrite-sync-api-server/main.go b/cmd/dendrite-sync-api-server/main.go index 55e9faeef..259447af2 100644 --- a/cmd/dendrite-sync-api-server/main.go +++ b/cmd/dendrite-sync-api-server/main.go @@ -28,9 +28,9 @@ func main() { accountDB := base.CreateAccountsDB() federation := base.CreateFederationClient() - _, _, query := base.CreateHTTPRoomserverAPIs() + rsAPI := base.CreateHTTPRoomserverAPIs() - syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg) + syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg) base.SetupAndServeHTTP(string(base.Cfg.Bind.SyncAPI), string(base.Cfg.Listen.SyncAPI)) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 1f2f20fb4..5b7ed4807 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -123,28 +123,28 @@ func main() { } p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node) - alias, input, query := roomserver.SetupRoomServerComponent(base, keyRing, federation) + rsAPI := roomserver.SetupRoomServerComponent(base, keyRing, federation) eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) asQuery := appservice.SetupAppServiceAPIComponent( - base, accountDB, deviceDB, federation, alias, query, transactions.New(), + base, accountDB, deviceDB, federation, rsAPI, transactions.New(), ) - fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query, input, &keyRing) - input.SetFederationSenderAPI(fedSenderAPI) + fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, rsAPI, &keyRing) + rsAPI.SetFederationSenderAPI(fedSenderAPI) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, - federation, &keyRing, alias, input, query, + federation, &keyRing, rsAPI, eduInputAPI, asQuery, transactions.New(), fedSenderAPI, ) eduProducer := producers.NewEDUServerProducer(eduInputAPI) - federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer) + federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asQuery, fedSenderAPI, eduProducer) mediaapi.SetupMediaAPIComponent(base, deviceDB) publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI)) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, query, federation, p2pPublicRoomProvider) - syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg) + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, federation, p2pPublicRoomProvider) + syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg) httpHandler := common.WrapHandlerInCORS(base.APIMux) diff --git a/cmd/roomserver-integration-tests/main.go b/cmd/roomserver-integration-tests/main.go index 682fc6224..7126405e5 100644 --- a/cmd/roomserver-integration-tests/main.go +++ b/cmd/roomserver-integration-tests/main.go @@ -209,7 +209,7 @@ func writeToRoomServer(input []string, roomserverURL string) error { return err } } - x, err := api.NewRoomserverInputAPIHTTP(roomserverURL, &http.Client{Timeout: timeoutHTTP}) + x, err := api.NewRoomserverInternalAPIHTTP(roomserverURL, &http.Client{Timeout: timeoutHTTP}, nil) if err != nil { return err } @@ -225,7 +225,7 @@ func writeToRoomServer(input []string, roomserverURL string) error { // Once those messages have been written it runs the checkQueries function passing // a api.RoomserverQueryAPI client. The caller can use this function to check the // behaviour of the query API. -func testRoomserver(input []string, wantOutput []string, checkQueries func(api.RoomserverQueryAPI)) { +func testRoomserver(input []string, wantOutput []string, checkQueries func(api.RoomserverInternalAPI)) { dir, err := ioutil.TempDir("", "room-server-test") if err != nil { panic(err) @@ -276,7 +276,7 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R cmd.Args = []string{"dendrite-room-server", "--config", filepath.Join(dir, test.ConfigFile)} gotOutput, err := runAndReadFromTopic(cmd, cfg.RoomServerURL()+"/metrics", doInput, outputTopic, len(wantOutput), func() { - queryAPI, _ := api.NewRoomserverQueryAPIHTTP("http://"+string(cfg.Listen.RoomServer), &http.Client{Timeout: timeoutHTTP}, cache) + queryAPI, _ := api.NewRoomserverInternalAPIHTTP("http://"+string(cfg.Listen.RoomServer), &http.Client{Timeout: timeoutHTTP}, cache) checkQueries(queryAPI) }) if err != nil { @@ -410,7 +410,7 @@ func main() { }}`, } - testRoomserver(input, want, func(q api.RoomserverQueryAPI) { + testRoomserver(input, want, func(q api.RoomserverInternalAPI) { var response api.QueryLatestEventsAndStateResponse if err := q.QueryLatestEventsAndState( context.Background(), diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index f245dd50b..154acd806 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -119,24 +119,12 @@ func (b *BaseDendrite) CreateHTTPAppServiceAPIs() appserviceAPI.AppServiceQueryA // CreateHTTPRoomserverAPIs returns the AliasAPI, InputAPI and QueryAPI for hitting // the roomserver over HTTP. -func (b *BaseDendrite) CreateHTTPRoomserverAPIs() ( - roomserverAPI.RoomserverAliasAPI, - roomserverAPI.RoomserverInputAPI, - roomserverAPI.RoomserverQueryAPI, -) { - alias, err := roomserverAPI.NewRoomserverAliasAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient) +func (b *BaseDendrite) CreateHTTPRoomserverAPIs() roomserverAPI.RoomserverInternalAPI { + rsAPI, err := roomserverAPI.NewRoomserverInternalAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient, b.ImmutableCache) if err != nil { - logrus.WithError(err).Panic("NewRoomserverAliasAPIHTTP failed") + logrus.WithError(err).Panic("NewRoomserverInternalAPIHTTP failed", b.httpClient) } - input, err := roomserverAPI.NewRoomserverInputAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient) - if err != nil { - logrus.WithError(err).Panic("NewRoomserverInputAPIHTTP failed", b.httpClient) - } - query, err := roomserverAPI.NewRoomserverQueryAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient, b.ImmutableCache) - if err != nil { - logrus.WithError(err).Panic("NewRoomserverQueryAPIHTTP failed", b.httpClient) - } - return alias, input, query + return rsAPI } // CreateHTTPEDUServerAPIs returns eduInputAPI for hitting the EDU diff --git a/common/events.go b/common/events.go index 556b7b671..0c8ead3a7 100644 --- a/common/events.go +++ b/common/events.go @@ -39,13 +39,13 @@ var ErrRoomNoExists = errors.New("Room does not exist") func BuildEvent( ctx context.Context, builder *gomatrixserverlib.EventBuilder, cfg *config.Dendrite, evTime time.Time, - queryAPI api.RoomserverQueryAPI, queryRes *api.QueryLatestEventsAndStateResponse, + rsAPI api.RoomserverInternalAPI, queryRes *api.QueryLatestEventsAndStateResponse, ) (*gomatrixserverlib.Event, error) { if queryRes == nil { queryRes = &api.QueryLatestEventsAndStateResponse{} } - err := AddPrevEventsToEvent(ctx, builder, queryAPI, queryRes) + err := AddPrevEventsToEvent(ctx, builder, rsAPI, queryRes) if err != nil { // This can pass through a ErrRoomNoExists to the caller return nil, err @@ -66,7 +66,7 @@ func BuildEvent( func AddPrevEventsToEvent( ctx context.Context, builder *gomatrixserverlib.EventBuilder, - queryAPI api.RoomserverQueryAPI, queryRes *api.QueryLatestEventsAndStateResponse, + rsAPI api.RoomserverInternalAPI, queryRes *api.QueryLatestEventsAndStateResponse, ) error { eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder) if err != nil { @@ -82,8 +82,8 @@ func AddPrevEventsToEvent( RoomID: builder.RoomID, StateToFetch: eventsNeeded.Tuples(), } - if err = queryAPI.QueryLatestEventsAndState(ctx, &queryReq, queryRes); err != nil { - return fmt.Errorf("queryAPI.QueryLatestEventsAndState: %w", err) + if err = rsAPI.QueryLatestEventsAndState(ctx, &queryReq, queryRes); err != nil { + return fmt.Errorf("rsAPI.QueryLatestEventsAndState: %w", err) } if !queryRes.RoomExists { diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index 72e2b54ac..d458b8537 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -36,18 +36,16 @@ func SetupFederationAPIComponent( deviceDB devices.Database, federation *gomatrixserverlib.FederationClient, keyRing *gomatrixserverlib.KeyRing, - aliasAPI roomserverAPI.RoomserverAliasAPI, - inputAPI roomserverAPI.RoomserverInputAPI, - queryAPI roomserverAPI.RoomserverQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, federationSenderAPI federationSenderAPI.FederationSenderInternalAPI, eduProducer *producers.EDUServerProducer, ) { - roomserverProducer := producers.NewRoomserverProducer(inputAPI, queryAPI) + roomserverProducer := producers.NewRoomserverProducer(rsAPI) routing.Setup( - base.APIMux, base.Cfg, queryAPI, aliasAPI, asAPI, - roomserverProducer, eduProducer, federationSenderAPI, *keyRing, + base.APIMux, base.Cfg, rsAPI, asAPI, roomserverProducer, + eduProducer, federationSenderAPI, *keyRing, federation, accountsDB, deviceDB, ) } diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index 6f49b9a81..651a4a2d2 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -33,7 +33,7 @@ import ( func Backfill( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID string, cfg *config.Dendrite, ) util.JSONResponse { @@ -82,7 +82,7 @@ func Backfill( } // Query the roomserver. - if err = query.QueryBackfill(httpReq.Context(), &req, &res); err != nil { + if err = rsAPI.QueryBackfill(httpReq.Context(), &req, &res); err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("query.QueryBackfill failed") return jsonerror.InternalServerError() } diff --git a/federationapi/routing/eventauth.go b/federationapi/routing/eventauth.go index 003165c85..34eaad1c5 100644 --- a/federationapi/routing/eventauth.go +++ b/federationapi/routing/eventauth.go @@ -25,13 +25,13 @@ import ( func GetEventAuth( ctx context.Context, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID string, eventID string, ) util.JSONResponse { // TODO: Optimisation: we shouldn't be querying all the room state // that is in state.StateEvents - we just ignore it. - state, err := getState(ctx, request, query, roomID, eventID) + state, err := getState(ctx, request, rsAPI, roomID, eventID) if err != nil { return *err } diff --git a/federationapi/routing/events.go b/federationapi/routing/events.go index 03492db45..ced9e3d53 100644 --- a/federationapi/routing/events.go +++ b/federationapi/routing/events.go @@ -29,11 +29,11 @@ import ( func GetEvent( ctx context.Context, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, eventID string, origin gomatrixserverlib.ServerName, ) util.JSONResponse { - event, err := getEvent(ctx, request, query, eventID) + event, err := getEvent(ctx, request, rsAPI, eventID) if err != nil { return *err } @@ -52,11 +52,11 @@ func GetEvent( func getEvent( ctx context.Context, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, eventID string, ) (*gomatrixserverlib.Event, *util.JSONResponse) { var authResponse api.QueryServerAllowedToSeeEventResponse - err := query.QueryServerAllowedToSeeEvent( + err := rsAPI.QueryServerAllowedToSeeEvent( ctx, &api.QueryServerAllowedToSeeEventRequest{ EventID: eventID, @@ -75,7 +75,7 @@ func getEvent( } var eventsResponse api.QueryEventsByIDResponse - err = query.QueryEventsByID( + err = rsAPI.QueryEventsByID( ctx, &api.QueryEventsByIDRequest{EventIDs: []string{eventID}}, &eventsResponse, diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 0c899ab9f..be5e988ab 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -33,13 +33,13 @@ func MakeJoin( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, cfg *config.Dendrite, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID, userID string, remoteVersions []gomatrixserverlib.RoomVersion, ) util.JSONResponse { verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := query.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { + if err := rsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { return util.JSONResponse{ Code: http.StatusInternalServerError, JSON: jsonerror.InternalServerError(), @@ -97,7 +97,7 @@ func MakeJoin( queryRes := api.QueryLatestEventsAndStateResponse{ RoomVersion: verRes.RoomVersion, } - event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), query, &queryRes) + event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes) if err == common.ErrRoomNoExists { return util.JSONResponse{ Code: http.StatusNotFound, @@ -136,15 +136,15 @@ func SendJoin( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, cfg *config.Dendrite, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, producer *producers.RoomserverProducer, keys gomatrixserverlib.KeyRing, roomID, eventID string, ) util.JSONResponse { verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := query.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { - util.GetLogger(httpReq.Context()).WithError(err).Error("query.QueryRoomVersionForRoom failed") + if err := rsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { + util.GetLogger(httpReq.Context()).WithError(err).Error("rsAPI.QueryRoomVersionForRoom failed") return util.JSONResponse{ Code: http.StatusInternalServerError, JSON: jsonerror.InternalServerError(), @@ -216,14 +216,14 @@ func SendJoin( // Fetch the state and auth chain. We do this before we send the events // on, in case this fails. var stateAndAuthChainResponse api.QueryStateAndAuthChainResponse - err = query.QueryStateAndAuthChain(httpReq.Context(), &api.QueryStateAndAuthChainRequest{ + err = rsAPI.QueryStateAndAuthChain(httpReq.Context(), &api.QueryStateAndAuthChainRequest{ PrevEventIDs: event.PrevEventIDs(), AuthEventIDs: event.AuthEventIDs(), RoomID: roomID, ResolveState: true, }, &stateAndAuthChainResponse) if err != nil { - util.GetLogger(httpReq.Context()).WithError(err).Error("query.QueryStateAndAuthChain failed") + util.GetLogger(httpReq.Context()).WithError(err).Error("rsAPI.QueryStateAndAuthChain failed") return jsonerror.InternalServerError() } diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index 6fc3b12ed..1124bfa27 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -30,7 +30,7 @@ func MakeLeave( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, cfg *config.Dendrite, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID, userID string, ) util.JSONResponse { _, domain, err := gomatrixserverlib.SplitID('@', userID) @@ -61,7 +61,7 @@ func MakeLeave( } var queryRes api.QueryLatestEventsAndStateResponse - event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), query, &queryRes) + event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes) if err == common.ErrRoomNoExists { return util.JSONResponse{ Code: http.StatusNotFound, @@ -102,7 +102,7 @@ func SendLeave( ) util.JSONResponse { verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := producer.QueryAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { + if err := producer.RsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UnsupportedRoomVersion(err.Error()), diff --git a/federationapi/routing/missingevents.go b/federationapi/routing/missingevents.go index 069bff3dd..ae91c5891 100644 --- a/federationapi/routing/missingevents.go +++ b/federationapi/routing/missingevents.go @@ -34,7 +34,7 @@ type getMissingEventRequest struct { func GetMissingEvents( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID string, ) util.JSONResponse { var gme getMissingEventRequest @@ -46,7 +46,7 @@ func GetMissingEvents( } var eventsResponse api.QueryMissingEventsResponse - if err := query.QueryMissingEvents( + if err := rsAPI.QueryMissingEvents( httpReq.Context(), &api.QueryMissingEventsRequest{ EarliestEvents: gme.EarliestEvents, LatestEvents: gme.LatestEvents, diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go index 13c92451d..c58690c63 100644 --- a/federationapi/routing/query.go +++ b/federationapi/routing/query.go @@ -32,7 +32,7 @@ func RoomAliasToID( httpReq *http.Request, federation *gomatrixserverlib.FederationClient, cfg *config.Dendrite, - aliasAPI roomserverAPI.RoomserverAliasAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, senderAPI federationSenderAPI.FederationSenderInternalAPI, ) util.JSONResponse { roomAlias := httpReq.FormValue("room_alias") @@ -55,7 +55,7 @@ func RoomAliasToID( if domain == cfg.Matrix.ServerName { queryReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: roomAlias} var queryRes roomserverAPI.GetRoomIDForAliasResponse - if err = aliasAPI.GetRoomIDForAlias(httpReq.Context(), &queryReq, &queryRes); err != nil { + if err = rsAPI.GetRoomIDForAlias(httpReq.Context(), &queryReq, &queryRes); err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("aliasAPI.GetRoomIDForAlias failed") return jsonerror.InternalServerError() } diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index e0f842f18..a5b8ce24e 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -44,8 +44,7 @@ const ( func Setup( apiMux *mux.Router, cfg *config.Dendrite, - query roomserverAPI.RoomserverQueryAPI, - aliasAPI roomserverAPI.RoomserverAliasAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, producer *producers.RoomserverProducer, eduProducer *producers.EDUServerProducer, @@ -80,7 +79,7 @@ func Setup( } return Send( httpReq, request, gomatrixserverlib.TransactionID(vars["txnID"]), - cfg, query, producer, eduProducer, keys, federation, + cfg, rsAPI, producer, eduProducer, keys, federation, ) }, )).Methods(http.MethodPut, http.MethodOptions) @@ -101,7 +100,7 @@ func Setup( v1fedmux.Handle("/3pid/onbind", common.MakeExternalAPI("3pid_onbind", func(req *http.Request) util.JSONResponse { - return CreateInvitesFrom3PIDInvites(req, query, asAPI, cfg, producer, federation, accountDB) + return CreateInvitesFrom3PIDInvites(req, rsAPI, asAPI, cfg, producer, federation, accountDB) }, )).Methods(http.MethodPost, http.MethodOptions) @@ -113,7 +112,7 @@ func Setup( return util.ErrorResponse(err) } return ExchangeThirdPartyInvite( - httpReq, request, vars["roomID"], query, cfg, federation, producer, + httpReq, request, vars["roomID"], rsAPI, cfg, federation, producer, ) }, )).Methods(http.MethodPut, http.MethodOptions) @@ -126,7 +125,7 @@ func Setup( return util.ErrorResponse(err) } return GetEvent( - httpReq.Context(), request, query, vars["eventID"], cfg.Matrix.ServerName, + httpReq.Context(), request, rsAPI, vars["eventID"], cfg.Matrix.ServerName, ) }, )).Methods(http.MethodGet) @@ -139,7 +138,7 @@ func Setup( return util.ErrorResponse(err) } return GetState( - httpReq.Context(), request, query, vars["roomID"], + httpReq.Context(), request, rsAPI, vars["roomID"], ) }, )).Methods(http.MethodGet) @@ -152,7 +151,7 @@ func Setup( return util.ErrorResponse(err) } return GetStateIDs( - httpReq.Context(), request, query, vars["roomID"], + httpReq.Context(), request, rsAPI, vars["roomID"], ) }, )).Methods(http.MethodGet) @@ -162,7 +161,7 @@ func Setup( func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars := mux.Vars(httpReq) return GetEventAuth( - httpReq.Context(), request, query, vars["roomID"], vars["eventID"], + httpReq.Context(), request, rsAPI, vars["roomID"], vars["eventID"], ) }, )).Methods(http.MethodGet) @@ -171,7 +170,7 @@ func Setup( "federation_query_room_alias", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { return RoomAliasToID( - httpReq, federation, cfg, aliasAPI, federationSenderAPI, + httpReq, federation, cfg, rsAPI, federationSenderAPI, ) }, )).Methods(http.MethodGet) @@ -222,7 +221,7 @@ func Setup( remoteVersions = append(remoteVersions, gomatrixserverlib.RoomVersionV1) } return MakeJoin( - httpReq, request, cfg, query, roomID, eventID, remoteVersions, + httpReq, request, cfg, rsAPI, roomID, eventID, remoteVersions, ) }, )).Methods(http.MethodGet) @@ -237,7 +236,7 @@ func Setup( roomID := vars["roomID"] eventID := vars["eventID"] return SendJoin( - httpReq, request, cfg, query, producer, keys, roomID, eventID, + httpReq, request, cfg, rsAPI, producer, keys, roomID, eventID, ) }, )).Methods(http.MethodPut) @@ -252,7 +251,7 @@ func Setup( roomID := vars["roomID"] eventID := vars["eventID"] return MakeLeave( - httpReq, request, cfg, query, roomID, eventID, + httpReq, request, cfg, rsAPI, roomID, eventID, ) }, )).Methods(http.MethodGet) @@ -286,7 +285,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetMissingEvents(httpReq, request, query, vars["roomID"]) + return GetMissingEvents(httpReq, request, rsAPI, vars["roomID"]) }, )).Methods(http.MethodPost) @@ -297,7 +296,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return Backfill(httpReq, request, query, vars["roomID"], cfg) + return Backfill(httpReq, request, rsAPI, vars["roomID"], cfg) }, )).Methods(http.MethodGet) } diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 5a9766f81..88411b81a 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -34,7 +34,7 @@ func Send( request *gomatrixserverlib.FederationRequest, txnID gomatrixserverlib.TransactionID, cfg *config.Dendrite, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, producer *producers.RoomserverProducer, eduProducer *producers.EDUServerProducer, keys gomatrixserverlib.KeyRing, @@ -42,7 +42,7 @@ func Send( ) util.JSONResponse { t := txnReq{ context: httpReq.Context(), - query: query, + rsAPI: rsAPI, producer: producer, eduProducer: eduProducer, keys: keys, @@ -99,7 +99,7 @@ func Send( type txnReq struct { gomatrixserverlib.Transaction context context.Context - query api.RoomserverQueryAPI + rsAPI api.RoomserverInternalAPI producer *producers.RoomserverProducer eduProducer *producers.EDUServerProducer keys gomatrixserverlib.KeyRing @@ -120,7 +120,7 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { } verReq := api.QueryRoomVersionForRoomRequest{RoomID: header.RoomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := t.query.QueryRoomVersionForRoom(t.context, &verReq, &verRes); err != nil { + if err := t.rsAPI.QueryRoomVersionForRoom(t.context, &verReq, &verRes); err != nil { util.GetLogger(t.context).WithError(err).Warn("Transaction: Failed to query room version for room", verReq.RoomID) return nil, roomNotFoundError{verReq.RoomID} } @@ -228,7 +228,7 @@ func (t *txnReq) processEvent(e gomatrixserverlib.Event) error { StateToFetch: needed.Tuples(), } var stateResp api.QueryStateAfterEventsResponse - if err := t.query.QueryStateAfterEvents(t.context, &stateReq, &stateResp); err != nil { + if err := t.rsAPI.QueryStateAfterEvents(t.context, &stateReq, &stateResp); err != nil { return err } diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go index 548598dd7..f90c494c3 100644 --- a/federationapi/routing/state.go +++ b/federationapi/routing/state.go @@ -27,7 +27,7 @@ import ( func GetState( ctx context.Context, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID string, ) util.JSONResponse { eventID, err := parseEventIDParam(request) @@ -35,7 +35,7 @@ func GetState( return *err } - state, err := getState(ctx, request, query, roomID, eventID) + state, err := getState(ctx, request, rsAPI, roomID, eventID) if err != nil { return *err } @@ -47,7 +47,7 @@ func GetState( func GetStateIDs( ctx context.Context, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID string, ) util.JSONResponse { eventID, err := parseEventIDParam(request) @@ -55,7 +55,7 @@ func GetStateIDs( return *err } - state, err := getState(ctx, request, query, roomID, eventID) + state, err := getState(ctx, request, rsAPI, roomID, eventID) if err != nil { return *err } @@ -94,11 +94,11 @@ func parseEventIDParam( func getState( ctx context.Context, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID string, eventID string, ) (*gomatrixserverlib.RespState, *util.JSONResponse) { - event, resErr := getEvent(ctx, request, query, eventID) + event, resErr := getEvent(ctx, request, rsAPI, eventID) if resErr != nil { return nil, resErr } @@ -110,7 +110,7 @@ func getState( authEventIDs := getIDsFromEventRef(event.AuthEvents()) var response api.QueryStateAndAuthChainResponse - err := query.QueryStateAndAuthChain( + err := rsAPI.QueryStateAndAuthChain( ctx, &api.QueryStateAndAuthChainRequest{ RoomID: roomID, diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index f93d934ed..3c1d09e11 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -58,7 +58,7 @@ var ( // CreateInvitesFrom3PIDInvites implements POST /_matrix/federation/v1/3pid/onbind func CreateInvitesFrom3PIDInvites( - req *http.Request, queryAPI roomserverAPI.RoomserverQueryAPI, + req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, cfg *config.Dendrite, producer *producers.RoomserverProducer, federation *gomatrixserverlib.FederationClient, accountDB accounts.Database, @@ -72,7 +72,7 @@ func CreateInvitesFrom3PIDInvites( for _, inv := range body.Invites { verReq := api.QueryRoomVersionForRoomRequest{RoomID: inv.RoomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := queryAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { + if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UnsupportedRoomVersion(err.Error()), @@ -80,7 +80,7 @@ func CreateInvitesFrom3PIDInvites( } event, err := createInviteFrom3PIDInvite( - req.Context(), queryAPI, asAPI, cfg, inv, federation, accountDB, + req.Context(), rsAPI, asAPI, cfg, inv, federation, accountDB, ) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("createInviteFrom3PIDInvite failed") @@ -108,7 +108,7 @@ func ExchangeThirdPartyInvite( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, roomID string, - queryAPI roomserverAPI.RoomserverQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, cfg *config.Dendrite, federation *gomatrixserverlib.FederationClient, producer *producers.RoomserverProducer, @@ -148,7 +148,7 @@ func ExchangeThirdPartyInvite( verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err = queryAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { + if err = rsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UnsupportedRoomVersion(err.Error()), @@ -156,7 +156,7 @@ func ExchangeThirdPartyInvite( } // Auth and build the event from what the remote server sent us - event, err := buildMembershipEvent(httpReq.Context(), &builder, queryAPI, cfg) + event, err := buildMembershipEvent(httpReq.Context(), &builder, rsAPI, cfg) if err == errNotInRoom { return util.JSONResponse{ Code: http.StatusNotFound, @@ -199,14 +199,14 @@ func ExchangeThirdPartyInvite( // Returns an error if there was a problem building the event or fetching the // necessary data to do so. func createInviteFrom3PIDInvite( - ctx context.Context, queryAPI roomserverAPI.RoomserverQueryAPI, + ctx context.Context, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, cfg *config.Dendrite, inv invite, federation *gomatrixserverlib.FederationClient, accountDB accounts.Database, ) (*gomatrixserverlib.Event, error) { verReq := api.QueryRoomVersionForRoomRequest{RoomID: inv.RoomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := queryAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil { + if err := rsAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil { return nil, err } @@ -245,7 +245,7 @@ func createInviteFrom3PIDInvite( return nil, err } - event, err := buildMembershipEvent(ctx, builder, queryAPI, cfg) + event, err := buildMembershipEvent(ctx, builder, rsAPI, cfg) if err == errNotInRoom { return nil, sendToRemoteServer(ctx, inv, federation, cfg, *builder) } @@ -263,7 +263,7 @@ func createInviteFrom3PIDInvite( // Returns an error if something failed during the process. func buildMembershipEvent( ctx context.Context, - builder *gomatrixserverlib.EventBuilder, queryAPI roomserverAPI.RoomserverQueryAPI, + builder *gomatrixserverlib.EventBuilder, rsAPI roomserverAPI.RoomserverInternalAPI, cfg *config.Dendrite, ) (*gomatrixserverlib.Event, error) { eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder) @@ -281,7 +281,7 @@ func buildMembershipEvent( StateToFetch: eventsNeeded.Tuples(), } var queryRes roomserverAPI.QueryLatestEventsAndStateResponse - if err = queryAPI.QueryLatestEventsAndState(ctx, &queryReq, &queryRes); err != nil { + if err = rsAPI.QueryLatestEventsAndState(ctx, &queryReq, &queryRes); err != nil { return nil, err } diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index 18c8324b4..67d08b339 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -33,11 +33,11 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { - cfg *config.Dendrite - roomServerConsumer *common.ContinualConsumer - db storage.Database - queues *queue.OutgoingQueues - query api.RoomserverQueryAPI + cfg *config.Dendrite + rsAPI api.RoomserverInternalAPI + rsConsumer *common.ContinualConsumer + db storage.Database + queues *queue.OutgoingQueues } // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. @@ -46,7 +46,7 @@ func NewOutputRoomEventConsumer( kafkaConsumer sarama.Consumer, queues *queue.OutgoingQueues, store storage.Database, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, ) *OutputRoomEventConsumer { consumer := common.ContinualConsumer{ Topic: string(cfg.Kafka.Topics.OutputRoomEvent), @@ -54,11 +54,11 @@ func NewOutputRoomEventConsumer( PartitionStore: store, } s := &OutputRoomEventConsumer{ - cfg: cfg, - roomServerConsumer: &consumer, - db: store, - queues: queues, - query: queryAPI, + cfg: cfg, + rsConsumer: &consumer, + db: store, + queues: queues, + rsAPI: rsAPI, } consumer.ProcessMessage = s.onMessage @@ -67,7 +67,7 @@ func NewOutputRoomEventConsumer( // Start consuming from room servers func (s *OutputRoomEventConsumer) Start() error { - return s.roomServerConsumer.Start() + return s.rsConsumer.Start() } // onMessage is called when the federation server receives a new event from the room server output log. @@ -369,7 +369,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( // from the roomserver using the query API. eventReq := api.QueryEventsByIDRequest{EventIDs: missing} var eventResp api.QueryEventsByIDResponse - if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { + if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { return nil, err } diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index aa9a7bc9c..bf9d326bb 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -34,8 +34,7 @@ import ( func SetupFederationSenderComponent( base *basecomponent.BaseDendrite, federation *gomatrixserverlib.FederationClient, - rsQueryAPI roomserverAPI.RoomserverQueryAPI, - rsInputAPI roomserverAPI.RoomserverInputAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, keyRing *gomatrixserverlib.KeyRing, ) api.FederationSenderInternalAPI { federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender)) @@ -43,13 +42,13 @@ func SetupFederationSenderComponent( logrus.WithError(err).Panic("failed to connect to federation sender db") } - roomserverProducer := producers.NewRoomserverProducer(rsInputAPI, base.Cfg.Matrix.ServerName) + roomserverProducer := producers.NewRoomserverProducer(rsAPI, base.Cfg.Matrix.ServerName) queues := queue.NewOutgoingQueues(base.Cfg.Matrix.ServerName, federation, roomserverProducer) rsConsumer := consumers.NewOutputRoomEventConsumer( base.Cfg, base.KafkaConsumer, queues, - federationSenderDB, rsQueryAPI, + federationSenderDB, rsAPI, ) if err = rsConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start room server consumer") diff --git a/federationsender/producers/roomserver.go b/federationsender/producers/roomserver.go index ff4cda5b5..48aeed8cc 100644 --- a/federationsender/producers/roomserver.go +++ b/federationsender/producers/roomserver.go @@ -23,16 +23,16 @@ import ( // RoomserverProducer produces events for the roomserver to consume. type RoomserverProducer struct { - InputAPI api.RoomserverInputAPI + InputAPI api.RoomserverInternalAPI serverName gomatrixserverlib.ServerName } // NewRoomserverProducer creates a new RoomserverProducer func NewRoomserverProducer( - inputAPI api.RoomserverInputAPI, serverName gomatrixserverlib.ServerName, + rsAPI api.RoomserverInternalAPI, serverName gomatrixserverlib.ServerName, ) *RoomserverProducer { return &RoomserverProducer{ - InputAPI: inputAPI, + InputAPI: rsAPI, serverName: serverName, } } diff --git a/publicroomsapi/consumers/roomserver.go b/publicroomsapi/consumers/roomserver.go index 853761c36..efd093b7e 100644 --- a/publicroomsapi/consumers/roomserver.go +++ b/publicroomsapi/consumers/roomserver.go @@ -29,9 +29,9 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { - roomServerConsumer *common.ContinualConsumer - db storage.Database - query api.RoomserverQueryAPI + rsAPI api.RoomserverInternalAPI + rsConsumer *common.ContinualConsumer + db storage.Database } // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. @@ -39,7 +39,7 @@ func NewOutputRoomEventConsumer( cfg *config.Dendrite, kafkaConsumer sarama.Consumer, store storage.Database, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, ) *OutputRoomEventConsumer { consumer := common.ContinualConsumer{ Topic: string(cfg.Kafka.Topics.OutputRoomEvent), @@ -47,9 +47,9 @@ func NewOutputRoomEventConsumer( PartitionStore: store, } s := &OutputRoomEventConsumer{ - roomServerConsumer: &consumer, - db: store, - query: queryAPI, + rsConsumer: &consumer, + db: store, + rsAPI: rsAPI, } consumer.ProcessMessage = s.onMessage @@ -58,7 +58,7 @@ func NewOutputRoomEventConsumer( // Start consuming from room servers func (s *OutputRoomEventConsumer) Start() error { - return s.roomServerConsumer.Start() + return s.rsConsumer.Start() } // onMessage is called when the sync server receives a new event from the room server output log. @@ -87,14 +87,14 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { addQueryReq := api.QueryEventsByIDRequest{EventIDs: output.NewRoomEvent.AddsStateEventIDs} var addQueryRes api.QueryEventsByIDResponse - if err := s.query.QueryEventsByID(context.TODO(), &addQueryReq, &addQueryRes); err != nil { + if err := s.rsAPI.QueryEventsByID(context.TODO(), &addQueryReq, &addQueryRes); err != nil { log.Warn(err) return err } remQueryReq := api.QueryEventsByIDRequest{EventIDs: output.NewRoomEvent.RemovesStateEventIDs} var remQueryRes api.QueryEventsByIDResponse - if err := s.query.QueryEventsByID(context.TODO(), &remQueryReq, &remQueryRes); err != nil { + if err := s.rsAPI.QueryEventsByID(context.TODO(), &remQueryReq, &remQueryRes); err != nil { log.Warn(err) return err } diff --git a/publicroomsapi/directory/directory.go b/publicroomsapi/directory/directory.go index 837018e64..fe7a67932 100644 --- a/publicroomsapi/directory/directory.go +++ b/publicroomsapi/directory/directory.go @@ -59,7 +59,7 @@ func GetVisibility( // SetVisibility implements PUT /directory/list/room/{roomID} // TODO: Allow admin users to edit the room visibility func SetVisibility( - req *http.Request, publicRoomsDatabase storage.Database, queryAPI api.RoomserverQueryAPI, dev *authtypes.Device, + req *http.Request, publicRoomsDatabase storage.Database, rsAPI api.RoomserverInternalAPI, dev *authtypes.Device, roomID string, ) util.JSONResponse { queryMembershipReq := api.QueryMembershipForUserRequest{ @@ -67,7 +67,7 @@ func SetVisibility( UserID: dev.UserID, } var queryMembershipRes api.QueryMembershipForUserResponse - err := queryAPI.QueryMembershipForUser(req.Context(), &queryMembershipReq, &queryMembershipRes) + err := rsAPI.QueryMembershipForUser(req.Context(), &queryMembershipReq, &queryMembershipRes) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("could not query membership for user") return jsonerror.InternalServerError() @@ -87,7 +87,7 @@ func SetVisibility( }}, } var queryEventsRes api.QueryLatestEventsAndStateResponse - err = queryAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes) + err = rsAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes) if err != nil || len(queryEventsRes.StateEvents) == 0 { util.GetLogger(req.Context()).WithError(err).Error("could not query events from room") return jsonerror.InternalServerError() diff --git a/publicroomsapi/publicroomsapi.go b/publicroomsapi/publicroomsapi.go index 6efb54bd9..6a4e65674 100644 --- a/publicroomsapi/publicroomsapi.go +++ b/publicroomsapi/publicroomsapi.go @@ -32,16 +32,16 @@ func SetupPublicRoomsAPIComponent( base *basecomponent.BaseDendrite, deviceDB devices.Database, publicRoomsDB storage.Database, - rsQueryAPI roomserverAPI.RoomserverQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider, ) { rsConsumer := consumers.NewOutputRoomEventConsumer( - base.Cfg, base.KafkaConsumer, publicRoomsDB, rsQueryAPI, + base.Cfg, base.KafkaConsumer, publicRoomsDB, rsAPI, ) if err := rsConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start public rooms server consumer") } - routing.Setup(base.APIMux, deviceDB, publicRoomsDB, rsQueryAPI, fedClient, extRoomsProvider) + routing.Setup(base.APIMux, deviceDB, publicRoomsDB, rsAPI, fedClient, extRoomsProvider) } diff --git a/publicroomsapi/routing/routing.go b/publicroomsapi/routing/routing.go index da5ea90d6..09a8eff76 100644 --- a/publicroomsapi/routing/routing.go +++ b/publicroomsapi/routing/routing.go @@ -39,7 +39,7 @@ const pathPrefixR0 = "/_matrix/client/r0" // applied: // nolint: gocyclo func Setup( - apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database, queryAPI api.RoomserverQueryAPI, + apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database, rsAPI api.RoomserverInternalAPI, fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider, ) { r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() @@ -66,7 +66,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return directory.SetVisibility(req, publicRoomsDB, queryAPI, device, vars["roomID"]) + return directory.SetVisibility(req, publicRoomsDB, rsAPI, device, vars["roomID"]) }), ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/publicRooms", diff --git a/roomserver/alias/alias_test.go b/roomserver/alias/alias_test.go deleted file mode 100644 index 0aefa19d9..000000000 --- a/roomserver/alias/alias_test.go +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright 2019 Serra Allgood -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package alias - -import ( - "context" - "fmt" - "strings" - "testing" - - appserviceAPI "github.com/matrix-org/dendrite/appservice/api" - roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/gomatrixserverlib" -) - -type MockRoomserverAliasAPIDatabase struct { - mode string - attempts int -} - -// These methods can be essentially noop -func (db MockRoomserverAliasAPIDatabase) SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error { - return nil -} - -func (db MockRoomserverAliasAPIDatabase) GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error) { - aliases := make([]string, 0) - return aliases, nil -} - -func (db MockRoomserverAliasAPIDatabase) RemoveRoomAlias(ctx context.Context, alias string) error { - return nil -} - -func (db *MockRoomserverAliasAPIDatabase) GetCreatorIDForAlias( - ctx context.Context, alias string, -) (string, error) { - return "", nil -} - -func (db *MockRoomserverAliasAPIDatabase) GetRoomVersionForRoom( - ctx context.Context, roomID string, -) (gomatrixserverlib.RoomVersion, error) { - return gomatrixserverlib.RoomVersionV1, nil -} - -// This method needs to change depending on test case -func (db *MockRoomserverAliasAPIDatabase) GetRoomIDForAlias( - ctx context.Context, - alias string, -) (string, error) { - switch db.mode { - case "empty": - return "", nil - case "error": - return "", fmt.Errorf("found an error from GetRoomIDForAlias") - case "found": - return "123", nil - case "emptyFound": - switch db.attempts { - case 0: - db.attempts = 1 - return "", nil - case 1: - db.attempts = 0 - return "123", nil - default: - return "", nil - } - default: - return "", fmt.Errorf("unknown option used") - } -} - -type MockAppServiceQueryAPI struct { - mode string -} - -// This method can be noop -func (q MockAppServiceQueryAPI) UserIDExists( - ctx context.Context, - req *appserviceAPI.UserIDExistsRequest, - resp *appserviceAPI.UserIDExistsResponse, -) error { - return nil -} - -func (q MockAppServiceQueryAPI) RoomAliasExists( - ctx context.Context, - req *appserviceAPI.RoomAliasExistsRequest, - resp *appserviceAPI.RoomAliasExistsResponse, -) error { - switch q.mode { - case "error": - return fmt.Errorf("found an error from RoomAliasExists") - case "found": - resp.AliasExists = true - return nil - case "empty": - resp.AliasExists = false - return nil - default: - return fmt.Errorf("Unknown option used") - } -} - -func TestGetRoomIDForAlias(t *testing.T) { - type arguments struct { - ctx context.Context - request *roomserverAPI.GetRoomIDForAliasRequest - response *roomserverAPI.GetRoomIDForAliasResponse - } - args := arguments{ - context.Background(), - &roomserverAPI.GetRoomIDForAliasRequest{}, - &roomserverAPI.GetRoomIDForAliasResponse{}, - } - type testCase struct { - name string - dbMode string - queryMode string - wantError bool - errorMsg string - want string - } - tt := []testCase{ - { - "found local alias", - "found", - "error", - false, - "", - "123", - }, - { - "found appservice alias", - "emptyFound", - "found", - false, - "", - "123", - }, - { - "error returned from DB", - "error", - "", - true, - "GetRoomIDForAlias", - "", - }, - { - "error returned from appserviceAPI", - "empty", - "error", - true, - "RoomAliasExists", - "", - }, - { - "no errors but no alias", - "empty", - "empty", - false, - "", - "", - }, - } - - setup := func(dbMode, queryMode string) *RoomserverAliasAPI { - mockAliasAPIDB := &MockRoomserverAliasAPIDatabase{dbMode, 0} - mockAppServiceQueryAPI := MockAppServiceQueryAPI{queryMode} - - return &RoomserverAliasAPI{ - DB: mockAliasAPIDB, - AppserviceAPI: mockAppServiceQueryAPI, - } - } - - for _, tc := range tt { - t.Run(tc.name, func(t *testing.T) { - aliasAPI := setup(tc.dbMode, tc.queryMode) - - err := aliasAPI.GetRoomIDForAlias(args.ctx, args.request, args.response) - if tc.wantError { - if err == nil { - t.Fatalf("Got no error; wanted error from %s", tc.errorMsg) - } else if !strings.Contains(err.Error(), tc.errorMsg) { - t.Fatalf("Got %s; wanted error from %s", err, tc.errorMsg) - } - } else if err != nil { - t.Fatalf("Got %s; wanted no error", err) - } else if args.response.RoomID != tc.want { - t.Errorf("Got '%s'; wanted '%s'", args.response.RoomID, tc.want) - } - }) - } -} diff --git a/roomserver/api/alias.go b/roomserver/api/alias.go index ad375a830..488e99ab8 100644 --- a/roomserver/api/alias.go +++ b/roomserver/api/alias.go @@ -16,8 +16,6 @@ package api import ( "context" - "errors" - "net/http" commonHTTP "github.com/matrix-org/dendrite/common/http" opentracing "github.com/opentracing/opentracing-go" @@ -86,44 +84,6 @@ type RemoveRoomAliasRequest struct { // RemoveRoomAliasResponse is a response to RemoveRoomAlias type RemoveRoomAliasResponse struct{} -// RoomserverAliasAPI is used to save, lookup or remove a room alias -type RoomserverAliasAPI interface { - // Set a room alias - SetRoomAlias( - ctx context.Context, - req *SetRoomAliasRequest, - response *SetRoomAliasResponse, - ) error - - // Get the room ID for an alias - GetRoomIDForAlias( - ctx context.Context, - req *GetRoomIDForAliasRequest, - response *GetRoomIDForAliasResponse, - ) error - - // Get all known aliases for a room ID - GetAliasesForRoomID( - ctx context.Context, - req *GetAliasesForRoomIDRequest, - response *GetAliasesForRoomIDResponse, - ) error - - // Get the user ID of the creator of an alias - GetCreatorIDForAlias( - ctx context.Context, - req *GetCreatorIDForAliasRequest, - response *GetCreatorIDForAliasResponse, - ) error - - // Remove a room alias - RemoveRoomAlias( - ctx context.Context, - req *RemoveRoomAliasRequest, - response *RemoveRoomAliasResponse, - ) error -} - // RoomserverSetRoomAliasPath is the HTTP path for the SetRoomAlias API. const RoomserverSetRoomAliasPath = "/api/roomserver/setRoomAlias" @@ -139,22 +99,8 @@ const RoomserverGetCreatorIDForAliasPath = "/api/roomserver/GetCreatorIDForAlias // RoomserverRemoveRoomAliasPath is the HTTP path for the RemoveRoomAlias API. const RoomserverRemoveRoomAliasPath = "/api/roomserver/removeRoomAlias" -// NewRoomserverAliasAPIHTTP creates a RoomserverAliasAPI implemented by talking to a HTTP POST API. -// If httpClient is nil an error is returned -func NewRoomserverAliasAPIHTTP(roomserverURL string, httpClient *http.Client) (RoomserverAliasAPI, error) { - if httpClient == nil { - return nil, errors.New("NewRoomserverAliasAPIHTTP: httpClient is ") - } - return &httpRoomserverAliasAPI{roomserverURL, httpClient}, nil -} - -type httpRoomserverAliasAPI struct { - roomserverURL string - httpClient *http.Client -} - // SetRoomAlias implements RoomserverAliasAPI -func (h *httpRoomserverAliasAPI) SetRoomAlias( +func (h *httpRoomserverInternalAPI) SetRoomAlias( ctx context.Context, request *SetRoomAliasRequest, response *SetRoomAliasResponse, @@ -167,7 +113,7 @@ func (h *httpRoomserverAliasAPI) SetRoomAlias( } // GetRoomIDForAlias implements RoomserverAliasAPI -func (h *httpRoomserverAliasAPI) GetRoomIDForAlias( +func (h *httpRoomserverInternalAPI) GetRoomIDForAlias( ctx context.Context, request *GetRoomIDForAliasRequest, response *GetRoomIDForAliasResponse, @@ -180,7 +126,7 @@ func (h *httpRoomserverAliasAPI) GetRoomIDForAlias( } // GetAliasesForRoomID implements RoomserverAliasAPI -func (h *httpRoomserverAliasAPI) GetAliasesForRoomID( +func (h *httpRoomserverInternalAPI) GetAliasesForRoomID( ctx context.Context, request *GetAliasesForRoomIDRequest, response *GetAliasesForRoomIDResponse, @@ -193,7 +139,7 @@ func (h *httpRoomserverAliasAPI) GetAliasesForRoomID( } // GetCreatorIDForAlias implements RoomserverAliasAPI -func (h *httpRoomserverAliasAPI) GetCreatorIDForAlias( +func (h *httpRoomserverInternalAPI) GetCreatorIDForAlias( ctx context.Context, request *GetCreatorIDForAliasRequest, response *GetCreatorIDForAliasResponse, @@ -206,7 +152,7 @@ func (h *httpRoomserverAliasAPI) GetCreatorIDForAlias( } // RemoveRoomAlias implements RoomserverAliasAPI -func (h *httpRoomserverAliasAPI) RemoveRoomAlias( +func (h *httpRoomserverInternalAPI) RemoveRoomAlias( ctx context.Context, request *RemoveRoomAliasRequest, response *RemoveRoomAliasResponse, diff --git a/roomserver/api/api.go b/roomserver/api/api.go new file mode 100644 index 000000000..c12dbddde --- /dev/null +++ b/roomserver/api/api.go @@ -0,0 +1,141 @@ +package api + +import ( + "context" + + fsAPI "github.com/matrix-org/dendrite/federationsender/api" +) + +// RoomserverInputAPI is used to write events to the room server. +type RoomserverInternalAPI interface { + // needed to avoid chicken and egg scenario when setting up the + // interdependencies between the roomserver and other input APIs + SetFederationSenderAPI(fsAPI fsAPI.FederationSenderInternalAPI) + + InputRoomEvents( + ctx context.Context, + request *InputRoomEventsRequest, + response *InputRoomEventsResponse, + ) error + + // Query the latest events and state for a room from the room server. + QueryLatestEventsAndState( + ctx context.Context, + request *QueryLatestEventsAndStateRequest, + response *QueryLatestEventsAndStateResponse, + ) error + + // Query the state after a list of events in a room from the room server. + QueryStateAfterEvents( + ctx context.Context, + request *QueryStateAfterEventsRequest, + response *QueryStateAfterEventsResponse, + ) error + + // Query a list of events by event ID. + QueryEventsByID( + ctx context.Context, + request *QueryEventsByIDRequest, + response *QueryEventsByIDResponse, + ) error + + // Query the membership event for an user for a room. + QueryMembershipForUser( + ctx context.Context, + request *QueryMembershipForUserRequest, + response *QueryMembershipForUserResponse, + ) error + + // Query a list of membership events for a room + QueryMembershipsForRoom( + ctx context.Context, + request *QueryMembershipsForRoomRequest, + response *QueryMembershipsForRoomResponse, + ) error + + // Query a list of invite event senders for a user in a room. + QueryInvitesForUser( + ctx context.Context, + request *QueryInvitesForUserRequest, + response *QueryInvitesForUserResponse, + ) error + + // Query whether a server is allowed to see an event + QueryServerAllowedToSeeEvent( + ctx context.Context, + request *QueryServerAllowedToSeeEventRequest, + response *QueryServerAllowedToSeeEventResponse, + ) error + + // Query missing events for a room from roomserver + QueryMissingEvents( + ctx context.Context, + request *QueryMissingEventsRequest, + response *QueryMissingEventsResponse, + ) error + + // Query to get state and auth chain for a (potentially hypothetical) event. + // Takes lists of PrevEventIDs and AuthEventsIDs and uses them to calculate + // the state and auth chain to return. + QueryStateAndAuthChain( + ctx context.Context, + request *QueryStateAndAuthChainRequest, + response *QueryStateAndAuthChainResponse, + ) error + + // Query a given amount (or less) of events prior to a given set of events. + QueryBackfill( + ctx context.Context, + request *QueryBackfillRequest, + response *QueryBackfillResponse, + ) error + + // Asks for the default room version as preferred by the server. + QueryRoomVersionCapabilities( + ctx context.Context, + request *QueryRoomVersionCapabilitiesRequest, + response *QueryRoomVersionCapabilitiesResponse, + ) error + + // Asks for the room version for a given room. + QueryRoomVersionForRoom( + ctx context.Context, + request *QueryRoomVersionForRoomRequest, + response *QueryRoomVersionForRoomResponse, + ) error + + // Set a room alias + SetRoomAlias( + ctx context.Context, + req *SetRoomAliasRequest, + response *SetRoomAliasResponse, + ) error + + // Get the room ID for an alias + GetRoomIDForAlias( + ctx context.Context, + req *GetRoomIDForAliasRequest, + response *GetRoomIDForAliasResponse, + ) error + + // Get all known aliases for a room ID + GetAliasesForRoomID( + ctx context.Context, + req *GetAliasesForRoomIDRequest, + response *GetAliasesForRoomIDResponse, + ) error + + // Get the user ID of the creator of an alias + GetCreatorIDForAlias( + ctx context.Context, + req *GetCreatorIDForAliasRequest, + response *GetCreatorIDForAliasResponse, + ) error + + // Remove a room alias + RemoveRoomAlias( + ctx context.Context, + req *RemoveRoomAliasRequest, + response *RemoveRoomAliasResponse, + ) error +} diff --git a/roomserver/api/http.go b/roomserver/api/http.go new file mode 100644 index 000000000..d643526bd --- /dev/null +++ b/roomserver/api/http.go @@ -0,0 +1,41 @@ +package api + +import ( + "errors" + "net/http" + + "github.com/matrix-org/dendrite/common/caching" + fsInputAPI "github.com/matrix-org/dendrite/federationsender/api" +) + +type httpRoomserverInternalAPI struct { + roomserverURL string + httpClient *http.Client + fsAPI fsInputAPI.FederationSenderInternalAPI + immutableCache caching.ImmutableCache +} + +// NewRoomserverInputAPIHTTP creates a RoomserverInputAPI implemented by talking to a HTTP POST API. +// If httpClient is nil an error is returned +func NewRoomserverInternalAPIHTTP( + roomserverURL string, + httpClient *http.Client, + //fsInputAPI fsAPI.FederationSenderInternalAPI, + immutableCache caching.ImmutableCache, +) (RoomserverInternalAPI, error) { + if httpClient == nil { + return nil, errors.New("NewRoomserverInternalAPIHTTP: httpClient is ") + } + return &httpRoomserverInternalAPI{ + roomserverURL: roomserverURL, + httpClient: httpClient, + immutableCache: immutableCache, + }, nil +} + +// SetFederationSenderInputAPI passes in a federation sender input API reference +// so that we can avoid the chicken-and-egg problem of both the roomserver input API +// and the federation sender input API being interdependent. +func (h *httpRoomserverInternalAPI) SetFederationSenderAPI(fsAPI fsInputAPI.FederationSenderInternalAPI) { + h.fsAPI = fsAPI +} diff --git a/roomserver/api/input.go b/roomserver/api/input.go index d9cffad27..8e8fdae4e 100644 --- a/roomserver/api/input.go +++ b/roomserver/api/input.go @@ -17,11 +17,8 @@ package api import ( "context" - "errors" - "net/http" commonHTTP "github.com/matrix-org/dendrite/common/http" - fsAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/gomatrixserverlib" opentracing "github.com/opentracing/opentracing-go" ) @@ -105,47 +102,11 @@ type InputRoomEventsResponse struct { EventID string `json:"event_id"` } -// RoomserverInputAPI is used to write events to the room server. -type RoomserverInputAPI interface { - // needed to avoid chicken and egg scenario when setting up the - // interdependencies between the roomserver and the FS input API - SetFederationSenderAPI(fsInputAPI fsAPI.FederationSenderInternalAPI) - InputRoomEvents( - ctx context.Context, - request *InputRoomEventsRequest, - response *InputRoomEventsResponse, - ) error -} - // RoomserverInputRoomEventsPath is the HTTP path for the InputRoomEvents API. const RoomserverInputRoomEventsPath = "/api/roomserver/inputRoomEvents" -// NewRoomserverInputAPIHTTP creates a RoomserverInputAPI implemented by talking to a HTTP POST API. -// If httpClient is nil an error is returned -func NewRoomserverInputAPIHTTP(roomserverURL string, httpClient *http.Client) (RoomserverInputAPI, error) { - if httpClient == nil { - return nil, errors.New("NewRoomserverInputAPIHTTP: httpClient is ") - } - return &httpRoomserverInputAPI{roomserverURL, httpClient, nil}, nil -} - -type httpRoomserverInputAPI struct { - roomserverURL string - httpClient *http.Client - // The federation sender API allows us to send federation - // requests from the new perform input requests, still TODO. - fsInputAPI fsAPI.FederationSenderInternalAPI -} - -// SetFederationSenderInputAPI passes in a federation sender input API reference -// so that we can avoid the chicken-and-egg problem of both the roomserver input API -// and the federation sender input API being interdependent. -func (h *httpRoomserverInputAPI) SetFederationSenderAPI(fsInputAPI fsAPI.FederationSenderInternalAPI) { - h.fsInputAPI = fsInputAPI -} - // InputRoomEvents implements RoomserverInputAPI -func (h *httpRoomserverInputAPI) InputRoomEvents( +func (h *httpRoomserverInternalAPI) InputRoomEvents( ctx context.Context, request *InputRoomEventsRequest, response *InputRoomEventsResponse, diff --git a/roomserver/api/query.go b/roomserver/api/query.go index 11fa5c9ca..cb7cbb86c 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -18,10 +18,7 @@ package api import ( "context" - "errors" - "net/http" - "github.com/matrix-org/dendrite/common/caching" commonHTTP "github.com/matrix-org/dendrite/common/http" "github.com/matrix-org/gomatrixserverlib" opentracing "github.com/opentracing/opentracing-go" @@ -264,95 +261,6 @@ type QueryRoomVersionForRoomResponse struct { RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` } -// RoomserverQueryAPI is used to query information from the room server. -type RoomserverQueryAPI interface { - // Query the latest events and state for a room from the room server. - QueryLatestEventsAndState( - ctx context.Context, - request *QueryLatestEventsAndStateRequest, - response *QueryLatestEventsAndStateResponse, - ) error - - // Query the state after a list of events in a room from the room server. - QueryStateAfterEvents( - ctx context.Context, - request *QueryStateAfterEventsRequest, - response *QueryStateAfterEventsResponse, - ) error - - // Query a list of events by event ID. - QueryEventsByID( - ctx context.Context, - request *QueryEventsByIDRequest, - response *QueryEventsByIDResponse, - ) error - - // Query the membership event for an user for a room. - QueryMembershipForUser( - ctx context.Context, - request *QueryMembershipForUserRequest, - response *QueryMembershipForUserResponse, - ) error - - // Query a list of membership events for a room - QueryMembershipsForRoom( - ctx context.Context, - request *QueryMembershipsForRoomRequest, - response *QueryMembershipsForRoomResponse, - ) error - - // Query a list of invite event senders for a user in a room. - QueryInvitesForUser( - ctx context.Context, - request *QueryInvitesForUserRequest, - response *QueryInvitesForUserResponse, - ) error - - // Query whether a server is allowed to see an event - QueryServerAllowedToSeeEvent( - ctx context.Context, - request *QueryServerAllowedToSeeEventRequest, - response *QueryServerAllowedToSeeEventResponse, - ) error - - // Query missing events for a room from roomserver - QueryMissingEvents( - ctx context.Context, - request *QueryMissingEventsRequest, - response *QueryMissingEventsResponse, - ) error - - // Query to get state and auth chain for a (potentially hypothetical) event. - // Takes lists of PrevEventIDs and AuthEventsIDs and uses them to calculate - // the state and auth chain to return. - QueryStateAndAuthChain( - ctx context.Context, - request *QueryStateAndAuthChainRequest, - response *QueryStateAndAuthChainResponse, - ) error - - // Query a given amount (or less) of events prior to a given set of events. - QueryBackfill( - ctx context.Context, - request *QueryBackfillRequest, - response *QueryBackfillResponse, - ) error - - // Asks for the default room version as preferred by the server. - QueryRoomVersionCapabilities( - ctx context.Context, - request *QueryRoomVersionCapabilitiesRequest, - response *QueryRoomVersionCapabilitiesResponse, - ) error - - // Asks for the room version for a given room. - QueryRoomVersionForRoom( - ctx context.Context, - request *QueryRoomVersionForRoomRequest, - response *QueryRoomVersionForRoomResponse, - ) error -} - // RoomserverQueryLatestEventsAndStatePath is the HTTP path for the QueryLatestEventsAndState API. const RoomserverQueryLatestEventsAndStatePath = "/api/roomserver/queryLatestEventsAndState" @@ -389,23 +297,8 @@ const RoomserverQueryRoomVersionCapabilitiesPath = "/api/roomserver/queryRoomVer // RoomserverQueryRoomVersionForRoomPath is the HTTP path for the QueryRoomVersionForRoom API const RoomserverQueryRoomVersionForRoomPath = "/api/roomserver/queryRoomVersionForRoom" -// NewRoomserverQueryAPIHTTP creates a RoomserverQueryAPI implemented by talking to a HTTP POST API. -// If httpClient is nil an error is returned -func NewRoomserverQueryAPIHTTP(roomserverURL string, httpClient *http.Client, cache caching.ImmutableCache) (RoomserverQueryAPI, error) { - if httpClient == nil { - return nil, errors.New("NewRoomserverQueryAPIHTTP: httpClient is ") - } - return &httpRoomserverQueryAPI{roomserverURL, httpClient, cache}, nil -} - -type httpRoomserverQueryAPI struct { - roomserverURL string - httpClient *http.Client - immutableCache caching.ImmutableCache -} - // QueryLatestEventsAndState implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryLatestEventsAndState( +func (h *httpRoomserverInternalAPI) QueryLatestEventsAndState( ctx context.Context, request *QueryLatestEventsAndStateRequest, response *QueryLatestEventsAndStateResponse, @@ -418,7 +311,7 @@ func (h *httpRoomserverQueryAPI) QueryLatestEventsAndState( } // QueryStateAfterEvents implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryStateAfterEvents( +func (h *httpRoomserverInternalAPI) QueryStateAfterEvents( ctx context.Context, request *QueryStateAfterEventsRequest, response *QueryStateAfterEventsResponse, @@ -431,7 +324,7 @@ func (h *httpRoomserverQueryAPI) QueryStateAfterEvents( } // QueryEventsByID implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryEventsByID( +func (h *httpRoomserverInternalAPI) QueryEventsByID( ctx context.Context, request *QueryEventsByIDRequest, response *QueryEventsByIDResponse, @@ -444,7 +337,7 @@ func (h *httpRoomserverQueryAPI) QueryEventsByID( } // QueryMembershipForUser implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryMembershipForUser( +func (h *httpRoomserverInternalAPI) QueryMembershipForUser( ctx context.Context, request *QueryMembershipForUserRequest, response *QueryMembershipForUserResponse, @@ -457,7 +350,7 @@ func (h *httpRoomserverQueryAPI) QueryMembershipForUser( } // QueryMembershipsForRoom implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryMembershipsForRoom( +func (h *httpRoomserverInternalAPI) QueryMembershipsForRoom( ctx context.Context, request *QueryMembershipsForRoomRequest, response *QueryMembershipsForRoomResponse, @@ -470,7 +363,7 @@ func (h *httpRoomserverQueryAPI) QueryMembershipsForRoom( } // QueryInvitesForUser implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryInvitesForUser( +func (h *httpRoomserverInternalAPI) QueryInvitesForUser( ctx context.Context, request *QueryInvitesForUserRequest, response *QueryInvitesForUserResponse, @@ -483,7 +376,7 @@ func (h *httpRoomserverQueryAPI) QueryInvitesForUser( } // QueryServerAllowedToSeeEvent implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryServerAllowedToSeeEvent( +func (h *httpRoomserverInternalAPI) QueryServerAllowedToSeeEvent( ctx context.Context, request *QueryServerAllowedToSeeEventRequest, response *QueryServerAllowedToSeeEventResponse, @@ -496,7 +389,7 @@ func (h *httpRoomserverQueryAPI) QueryServerAllowedToSeeEvent( } // QueryMissingEvents implements RoomServerQueryAPI -func (h *httpRoomserverQueryAPI) QueryMissingEvents( +func (h *httpRoomserverInternalAPI) QueryMissingEvents( ctx context.Context, request *QueryMissingEventsRequest, response *QueryMissingEventsResponse, @@ -509,7 +402,7 @@ func (h *httpRoomserverQueryAPI) QueryMissingEvents( } // QueryStateAndAuthChain implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryStateAndAuthChain( +func (h *httpRoomserverInternalAPI) QueryStateAndAuthChain( ctx context.Context, request *QueryStateAndAuthChainRequest, response *QueryStateAndAuthChainResponse, @@ -522,7 +415,7 @@ func (h *httpRoomserverQueryAPI) QueryStateAndAuthChain( } // QueryBackfill implements RoomServerQueryAPI -func (h *httpRoomserverQueryAPI) QueryBackfill( +func (h *httpRoomserverInternalAPI) QueryBackfill( ctx context.Context, request *QueryBackfillRequest, response *QueryBackfillResponse, @@ -535,7 +428,7 @@ func (h *httpRoomserverQueryAPI) QueryBackfill( } // QueryRoomVersionCapabilities implements RoomServerQueryAPI -func (h *httpRoomserverQueryAPI) QueryRoomVersionCapabilities( +func (h *httpRoomserverInternalAPI) QueryRoomVersionCapabilities( ctx context.Context, request *QueryRoomVersionCapabilitiesRequest, response *QueryRoomVersionCapabilitiesResponse, @@ -548,7 +441,7 @@ func (h *httpRoomserverQueryAPI) QueryRoomVersionCapabilities( } // QueryRoomVersionForRoom implements RoomServerQueryAPI -func (h *httpRoomserverQueryAPI) QueryRoomVersionForRoom( +func (h *httpRoomserverInternalAPI) QueryRoomVersionForRoom( ctx context.Context, request *QueryRoomVersionForRoomRequest, response *QueryRoomVersionForRoomResponse, diff --git a/roomserver/alias/alias.go b/roomserver/internal/alias.go similarity index 50% rename from roomserver/alias/alias.go rename to roomserver/internal/alias.go index eb606e5cd..4139582b6 100644 --- a/roomserver/alias/alias.go +++ b/roomserver/internal/alias.go @@ -12,25 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -package alias +package internal import ( "context" "encoding/json" "errors" - "net/http" "time" - appserviceAPI "github.com/matrix-org/dendrite/appservice/api" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" - roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" ) -// RoomserverAliasAPIDatabase has the storage APIs needed to implement the alias API. -type RoomserverAliasAPIDatabase interface { +// RoomserverInternalAPIDatabase has the storage APIs needed to implement the alias API. +type RoomserverInternalAPIDatabase interface { // Save a given room alias with the room ID it refers to. // Returns an error if there was a problem talking to the database. SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error @@ -52,20 +47,11 @@ type RoomserverAliasAPIDatabase interface { ) (gomatrixserverlib.RoomVersion, error) } -// RoomserverAliasAPI is an implementation of alias.RoomserverAliasAPI -type RoomserverAliasAPI struct { - DB RoomserverAliasAPIDatabase - Cfg *config.Dendrite - InputAPI roomserverAPI.RoomserverInputAPI - QueryAPI roomserverAPI.RoomserverQueryAPI - AppserviceAPI appserviceAPI.AppServiceQueryAPI -} - -// SetRoomAlias implements alias.RoomserverAliasAPI -func (r *RoomserverAliasAPI) SetRoomAlias( +// SetRoomAlias implements alias.RoomserverInternalAPI +func (r *RoomserverInternalAPI) SetRoomAlias( ctx context.Context, - request *roomserverAPI.SetRoomAliasRequest, - response *roomserverAPI.SetRoomAliasResponse, + request *api.SetRoomAliasRequest, + response *api.SetRoomAliasResponse, ) error { // Check if the alias isn't already referring to a room roomID, err := r.DB.GetRoomIDForAlias(ctx, request.Alias) @@ -91,11 +77,11 @@ func (r *RoomserverAliasAPI) SetRoomAlias( return r.sendUpdatedAliasesEvent(context.TODO(), request.UserID, request.RoomID) } -// GetRoomIDForAlias implements alias.RoomserverAliasAPI -func (r *RoomserverAliasAPI) GetRoomIDForAlias( +// GetRoomIDForAlias implements alias.RoomserverInternalAPI +func (r *RoomserverInternalAPI) GetRoomIDForAlias( ctx context.Context, - request *roomserverAPI.GetRoomIDForAliasRequest, - response *roomserverAPI.GetRoomIDForAliasResponse, + request *api.GetRoomIDForAliasRequest, + response *api.GetRoomIDForAliasResponse, ) error { // Look up the room ID in the database roomID, err := r.DB.GetRoomIDForAlias(ctx, request.Alias) @@ -103,32 +89,38 @@ func (r *RoomserverAliasAPI) GetRoomIDForAlias( return err } - if roomID == "" { - // No room found locally, try our application services by making a call to - // the appservice component - aliasReq := appserviceAPI.RoomAliasExistsRequest{Alias: request.Alias} - var aliasResp appserviceAPI.RoomAliasExistsResponse - if err = r.AppserviceAPI.RoomAliasExists(ctx, &aliasReq, &aliasResp); err != nil { - return err - } + /* + TODO: Why is this here? It creates an unnecessary dependency + from the roomserver to the appservice component, which should be + altogether optional. - if aliasResp.AliasExists { - roomID, err = r.DB.GetRoomIDForAlias(ctx, request.Alias) - if err != nil { + if roomID == "" { + // No room found locally, try our application services by making a call to + // the appservice component + aliasReq := appserviceAPI.RoomAliasExistsRequest{Alias: request.Alias} + var aliasResp appserviceAPI.RoomAliasExistsResponse + if err = r.AppserviceAPI.RoomAliasExists(ctx, &aliasReq, &aliasResp); err != nil { return err } + + if aliasResp.AliasExists { + roomID, err = r.DB.GetRoomIDForAlias(ctx, request.Alias) + if err != nil { + return err + } + } } - } + */ response.RoomID = roomID return nil } -// GetAliasesForRoomID implements alias.RoomserverAliasAPI -func (r *RoomserverAliasAPI) GetAliasesForRoomID( +// GetAliasesForRoomID implements alias.RoomserverInternalAPI +func (r *RoomserverInternalAPI) GetAliasesForRoomID( ctx context.Context, - request *roomserverAPI.GetAliasesForRoomIDRequest, - response *roomserverAPI.GetAliasesForRoomIDResponse, + request *api.GetAliasesForRoomIDRequest, + response *api.GetAliasesForRoomIDResponse, ) error { // Look up the aliases in the database for the given RoomID aliases, err := r.DB.GetAliasesForRoomID(ctx, request.RoomID) @@ -140,11 +132,11 @@ func (r *RoomserverAliasAPI) GetAliasesForRoomID( return nil } -// GetCreatorIDForAlias implements alias.RoomserverAliasAPI -func (r *RoomserverAliasAPI) GetCreatorIDForAlias( +// GetCreatorIDForAlias implements alias.RoomserverInternalAPI +func (r *RoomserverInternalAPI) GetCreatorIDForAlias( ctx context.Context, - request *roomserverAPI.GetCreatorIDForAliasRequest, - response *roomserverAPI.GetCreatorIDForAliasResponse, + request *api.GetCreatorIDForAliasRequest, + response *api.GetCreatorIDForAliasResponse, ) error { // Look up the aliases in the database for the given RoomID creatorID, err := r.DB.GetCreatorIDForAlias(ctx, request.Alias) @@ -156,11 +148,11 @@ func (r *RoomserverAliasAPI) GetCreatorIDForAlias( return nil } -// RemoveRoomAlias implements alias.RoomserverAliasAPI -func (r *RoomserverAliasAPI) RemoveRoomAlias( +// RemoveRoomAlias implements alias.RoomserverInternalAPI +func (r *RoomserverInternalAPI) RemoveRoomAlias( ctx context.Context, - request *roomserverAPI.RemoveRoomAliasRequest, - response *roomserverAPI.RemoveRoomAliasResponse, + request *api.RemoveRoomAliasRequest, + response *api.RemoveRoomAliasResponse, ) error { // Look up the room ID in the database roomID, err := r.DB.GetRoomIDForAlias(ctx, request.Alias) @@ -186,7 +178,7 @@ type roomAliasesContent struct { // Build the updated m.room.aliases event to send to the room after addition or // removal of an alias -func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent( +func (r *RoomserverInternalAPI) sendUpdatedAliasesEvent( ctx context.Context, userID string, roomID string, ) error { serverName := string(r.Cfg.Matrix.ServerName) @@ -222,12 +214,12 @@ func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent( if len(eventsNeeded.Tuples()) == 0 { return errors.New("expecting state tuples for event builder, got none") } - req := roomserverAPI.QueryLatestEventsAndStateRequest{ + req := api.QueryLatestEventsAndStateRequest{ RoomID: roomID, StateToFetch: eventsNeeded.Tuples(), } - var res roomserverAPI.QueryLatestEventsAndStateResponse - if err = r.QueryAPI.QueryLatestEventsAndState(ctx, &req, &res); err != nil { + var res api.QueryLatestEventsAndStateResponse + if err = r.QueryLatestEventsAndState(ctx, &req, &res); err != nil { return err } builder.Depth = res.Depth @@ -263,91 +255,17 @@ func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent( } // Create the request - ire := roomserverAPI.InputRoomEvent{ - Kind: roomserverAPI.KindNew, + ire := api.InputRoomEvent{ + Kind: api.KindNew, Event: event.Headered(roomVersion), AuthEventIDs: event.AuthEventIDs(), SendAsServer: serverName, } - inputReq := roomserverAPI.InputRoomEventsRequest{ - InputRoomEvents: []roomserverAPI.InputRoomEvent{ire}, + inputReq := api.InputRoomEventsRequest{ + InputRoomEvents: []api.InputRoomEvent{ire}, } - var inputRes roomserverAPI.InputRoomEventsResponse + var inputRes api.InputRoomEventsResponse // Send the request - return r.InputAPI.InputRoomEvents(ctx, &inputReq, &inputRes) -} - -// SetupHTTP adds the RoomserverAliasAPI handlers to the http.ServeMux. -func (r *RoomserverAliasAPI) SetupHTTP(servMux *http.ServeMux) { - servMux.Handle( - roomserverAPI.RoomserverSetRoomAliasPath, - common.MakeInternalAPI("setRoomAlias", func(req *http.Request) util.JSONResponse { - var request roomserverAPI.SetRoomAliasRequest - var response roomserverAPI.SetRoomAliasResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.SetRoomAlias(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - roomserverAPI.RoomserverGetRoomIDForAliasPath, - common.MakeInternalAPI("GetRoomIDForAlias", func(req *http.Request) util.JSONResponse { - var request roomserverAPI.GetRoomIDForAliasRequest - var response roomserverAPI.GetRoomIDForAliasResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.GetRoomIDForAlias(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - roomserverAPI.RoomserverGetCreatorIDForAliasPath, - common.MakeInternalAPI("GetCreatorIDForAlias", func(req *http.Request) util.JSONResponse { - var request roomserverAPI.GetCreatorIDForAliasRequest - var response roomserverAPI.GetCreatorIDForAliasResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.GetCreatorIDForAlias(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - roomserverAPI.RoomserverGetAliasesForRoomIDPath, - common.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse { - var request roomserverAPI.GetAliasesForRoomIDRequest - var response roomserverAPI.GetAliasesForRoomIDResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.GetAliasesForRoomID(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - roomserverAPI.RoomserverRemoveRoomAliasPath, - common.MakeInternalAPI("removeRoomAlias", func(req *http.Request) util.JSONResponse { - var request roomserverAPI.RemoveRoomAliasRequest - var response roomserverAPI.RemoveRoomAliasResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.RemoveRoomAlias(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) + return r.InputRoomEvents(ctx, &inputReq, &inputRes) } diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go new file mode 100644 index 000000000..d1c443f24 --- /dev/null +++ b/roomserver/internal/api.go @@ -0,0 +1,287 @@ +package internal + +import ( + "encoding/json" + "net/http" + "sync" + + "github.com/Shopify/sarama" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/common/caching" + "github.com/matrix-org/dendrite/common/config" + fsAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/storage" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" +) + +// RoomserverInternalAPI is an implementation of api.RoomserverInternalAPI +type RoomserverInternalAPI struct { + DB storage.Database + Cfg *config.Dendrite + Producer sarama.SyncProducer + ImmutableCache caching.ImmutableCache + ServerName gomatrixserverlib.ServerName + KeyRing gomatrixserverlib.JSONVerifier + FedClient *gomatrixserverlib.FederationClient + OutputRoomEventTopic string // Kafka topic for new output room events + mutex sync.Mutex // Protects calls to processRoomEvent + fsAPI fsAPI.FederationSenderInternalAPI +} + +// SetupHTTP adds the RoomserverInternalAPI handlers to the http.ServeMux. +// nolint: gocyclo +func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { + servMux.Handle(api.RoomserverInputRoomEventsPath, + common.MakeInternalAPI("inputRoomEvents", func(req *http.Request) util.JSONResponse { + var request api.InputRoomEventsRequest + var response api.InputRoomEventsResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := r.InputRoomEvents(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryLatestEventsAndStatePath, + common.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse { + var request api.QueryLatestEventsAndStateRequest + var response api.QueryLatestEventsAndStateResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryLatestEventsAndState(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryStateAfterEventsPath, + common.MakeInternalAPI("queryStateAfterEvents", func(req *http.Request) util.JSONResponse { + var request api.QueryStateAfterEventsRequest + var response api.QueryStateAfterEventsResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryStateAfterEvents(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryEventsByIDPath, + common.MakeInternalAPI("queryEventsByID", func(req *http.Request) util.JSONResponse { + var request api.QueryEventsByIDRequest + var response api.QueryEventsByIDResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryEventsByID(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryMembershipForUserPath, + common.MakeInternalAPI("QueryMembershipForUser", func(req *http.Request) util.JSONResponse { + var request api.QueryMembershipForUserRequest + var response api.QueryMembershipForUserResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryMembershipForUser(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryMembershipsForRoomPath, + common.MakeInternalAPI("queryMembershipsForRoom", func(req *http.Request) util.JSONResponse { + var request api.QueryMembershipsForRoomRequest + var response api.QueryMembershipsForRoomResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryMembershipsForRoom(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryInvitesForUserPath, + common.MakeInternalAPI("queryInvitesForUser", func(req *http.Request) util.JSONResponse { + var request api.QueryInvitesForUserRequest + var response api.QueryInvitesForUserResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryInvitesForUser(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryServerAllowedToSeeEventPath, + common.MakeInternalAPI("queryServerAllowedToSeeEvent", func(req *http.Request) util.JSONResponse { + var request api.QueryServerAllowedToSeeEventRequest + var response api.QueryServerAllowedToSeeEventResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryServerAllowedToSeeEvent(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryMissingEventsPath, + common.MakeInternalAPI("queryMissingEvents", func(req *http.Request) util.JSONResponse { + var request api.QueryMissingEventsRequest + var response api.QueryMissingEventsResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryMissingEvents(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryStateAndAuthChainPath, + common.MakeInternalAPI("queryStateAndAuthChain", func(req *http.Request) util.JSONResponse { + var request api.QueryStateAndAuthChainRequest + var response api.QueryStateAndAuthChainResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryStateAndAuthChain(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryBackfillPath, + common.MakeInternalAPI("QueryBackfill", func(req *http.Request) util.JSONResponse { + var request api.QueryBackfillRequest + var response api.QueryBackfillResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryBackfill(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryRoomVersionCapabilitiesPath, + common.MakeInternalAPI("QueryRoomVersionCapabilities", func(req *http.Request) util.JSONResponse { + var request api.QueryRoomVersionCapabilitiesRequest + var response api.QueryRoomVersionCapabilitiesResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryRoomVersionCapabilities(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryRoomVersionForRoomPath, + common.MakeInternalAPI("QueryRoomVersionForRoom", func(req *http.Request) util.JSONResponse { + var request api.QueryRoomVersionForRoomRequest + var response api.QueryRoomVersionForRoomResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryRoomVersionForRoom(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverSetRoomAliasPath, + common.MakeInternalAPI("setRoomAlias", func(req *http.Request) util.JSONResponse { + var request api.SetRoomAliasRequest + var response api.SetRoomAliasResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.SetRoomAlias(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverGetRoomIDForAliasPath, + common.MakeInternalAPI("GetRoomIDForAlias", func(req *http.Request) util.JSONResponse { + var request api.GetRoomIDForAliasRequest + var response api.GetRoomIDForAliasResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.GetRoomIDForAlias(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverGetCreatorIDForAliasPath, + common.MakeInternalAPI("GetCreatorIDForAlias", func(req *http.Request) util.JSONResponse { + var request api.GetCreatorIDForAliasRequest + var response api.GetCreatorIDForAliasResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.GetCreatorIDForAlias(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverGetAliasesForRoomIDPath, + common.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse { + var request api.GetAliasesForRoomIDRequest + var response api.GetAliasesForRoomIDResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.GetAliasesForRoomID(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverRemoveRoomAliasPath, + common.MakeInternalAPI("removeRoomAlias", func(req *http.Request) util.JSONResponse { + var request api.RemoveRoomAliasRequest + var response api.RemoveRoomAliasResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.RemoveRoomAlias(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) +} diff --git a/roomserver/input/input.go b/roomserver/internal/input.go similarity index 55% rename from roomserver/input/input.go rename to roomserver/internal/input.go index 20b6afc4b..19ebea660 100644 --- a/roomserver/input/input.go +++ b/roomserver/internal/input.go @@ -13,46 +13,27 @@ // limitations under the License. // Package input contains the code processes new room events -package input +package internal import ( "context" "encoding/json" - "net/http" - "sync" "github.com/Shopify/sarama" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/roomserver/storage" - "github.com/matrix-org/util" fsAPI "github.com/matrix-org/dendrite/federationsender/api" ) -// RoomserverInputAPI implements api.RoomserverInputAPI -type RoomserverInputAPI struct { - DB storage.Database - Producer sarama.SyncProducer - // The kafkaesque topic to output new room events to. - // This is the name used in kafka to identify the stream to write events to. - OutputRoomEventTopic string - // Protects calls to processRoomEvent - mutex sync.Mutex - // The federation sender API allows us to send federation - // requests from the new perform input requests, still TODO. - fsAPI fsAPI.FederationSenderInternalAPI -} - // SetFederationSenderInputAPI passes in a federation sender input API reference // so that we can avoid the chicken-and-egg problem of both the roomserver input API // and the federation sender input API being interdependent. -func (r *RoomserverInputAPI) SetFederationSenderAPI(fsAPI fsAPI.FederationSenderInternalAPI) { +func (r *RoomserverInternalAPI) SetFederationSenderAPI(fsAPI fsAPI.FederationSenderInternalAPI) { r.fsAPI = fsAPI } // WriteOutputEvents implements OutputRoomEventWriter -func (r *RoomserverInputAPI) WriteOutputEvents(roomID string, updates []api.OutputEvent) error { +func (r *RoomserverInternalAPI) WriteOutputEvents(roomID string, updates []api.OutputEvent) error { messages := make([]*sarama.ProducerMessage, len(updates)) for i := range updates { value, err := json.Marshal(updates[i]) @@ -68,8 +49,8 @@ func (r *RoomserverInputAPI) WriteOutputEvents(roomID string, updates []api.Outp return r.Producer.SendMessages(messages) } -// InputRoomEvents implements api.RoomserverInputAPI -func (r *RoomserverInputAPI) InputRoomEvents( +// InputRoomEvents implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) InputRoomEvents( ctx context.Context, request *api.InputRoomEventsRequest, response *api.InputRoomEventsResponse, @@ -89,20 +70,3 @@ func (r *RoomserverInputAPI) InputRoomEvents( } return nil } - -// SetupHTTP adds the RoomserverInputAPI handlers to the http.ServeMux. -func (r *RoomserverInputAPI) SetupHTTP(servMux *http.ServeMux) { - servMux.Handle(api.RoomserverInputRoomEventsPath, - common.MakeInternalAPI("inputRoomEvents", func(req *http.Request) util.JSONResponse { - var request api.InputRoomEventsRequest - var response api.InputRoomEventsResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.MessageResponse(http.StatusBadRequest, err.Error()) - } - if err := r.InputRoomEvents(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) -} diff --git a/roomserver/input/authevents.go b/roomserver/internal/input_authevents.go similarity index 99% rename from roomserver/input/authevents.go rename to roomserver/internal/input_authevents.go index 2c2e14b3a..e3828f566 100644 --- a/roomserver/input/authevents.go +++ b/roomserver/internal/input_authevents.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package input +package internal import ( "context" diff --git a/roomserver/input/authevents_test.go b/roomserver/internal/input_authevents_test.go similarity index 99% rename from roomserver/input/authevents_test.go rename to roomserver/internal/input_authevents_test.go index 0621a0842..6b981571b 100644 --- a/roomserver/input/authevents_test.go +++ b/roomserver/internal/input_authevents_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package input +package internal import ( "testing" diff --git a/roomserver/input/events.go b/roomserver/internal/input_events.go similarity index 99% rename from roomserver/input/events.go rename to roomserver/internal/input_events.go index 69828d9f9..6da63716c 100644 --- a/roomserver/input/events.go +++ b/roomserver/internal/input_events.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package input +package internal import ( "context" diff --git a/roomserver/input/latest_events.go b/roomserver/internal/input_latest_events.go similarity index 99% rename from roomserver/input/latest_events.go rename to roomserver/internal/input_latest_events.go index cac3968d9..42be0f408 100644 --- a/roomserver/input/latest_events.go +++ b/roomserver/internal/input_latest_events.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package input +package internal import ( "bytes" diff --git a/roomserver/input/membership.go b/roomserver/internal/input_membership.go similarity index 99% rename from roomserver/input/membership.go rename to roomserver/internal/input_membership.go index 351e63d61..cba75b4fc 100644 --- a/roomserver/input/membership.go +++ b/roomserver/internal/input_membership.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package input +package internal import ( "context" diff --git a/roomserver/query/query.go b/roomserver/internal/query.go similarity index 74% rename from roomserver/query/query.go rename to roomserver/internal/query.go index 6778ac280..98adc24b3 100644 --- a/roomserver/query/query.go +++ b/roomserver/internal/query.go @@ -14,16 +14,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package query +package internal import ( "context" - "encoding/json" "fmt" - "net/http" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/caching" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/auth" "github.com/matrix-org/dendrite/roomserver/state" @@ -35,17 +31,8 @@ import ( "github.com/sirupsen/logrus" ) -// RoomserverQueryAPI is an implementation of api.RoomserverQueryAPI -type RoomserverQueryAPI struct { - DB storage.Database - ImmutableCache caching.ImmutableCache - ServerName gomatrixserverlib.ServerName - KeyRing gomatrixserverlib.JSONVerifier - FedClient *gomatrixserverlib.FederationClient -} - -// QueryLatestEventsAndState implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryLatestEventsAndState( +// QueryLatestEventsAndState implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryLatestEventsAndState( ctx context.Context, request *api.QueryLatestEventsAndStateRequest, response *api.QueryLatestEventsAndStateResponse, @@ -104,8 +91,8 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( return nil } -// QueryStateAfterEvents implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryStateAfterEvents( +// QueryStateAfterEvents implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryStateAfterEvents( ctx context.Context, request *api.QueryStateAfterEventsRequest, response *api.QueryStateAfterEventsResponse, @@ -160,8 +147,8 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( return nil } -// QueryEventsByID implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryEventsByID( +// QueryEventsByID implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryEventsByID( ctx context.Context, request *api.QueryEventsByIDRequest, response *api.QueryEventsByIDResponse, @@ -195,7 +182,7 @@ func (r *RoomserverQueryAPI) QueryEventsByID( return nil } -func (r *RoomserverQueryAPI) loadStateEvents( +func (r *RoomserverInternalAPI) loadStateEvents( ctx context.Context, stateEntries []types.StateEntry, ) ([]gomatrixserverlib.Event, error) { eventNIDs := make([]types.EventNID, len(stateEntries)) @@ -205,7 +192,7 @@ func (r *RoomserverQueryAPI) loadStateEvents( return r.loadEvents(ctx, eventNIDs) } -func (r *RoomserverQueryAPI) loadEvents( +func (r *RoomserverInternalAPI) loadEvents( ctx context.Context, eventNIDs []types.EventNID, ) ([]gomatrixserverlib.Event, error) { stateEvents, err := r.DB.Events(ctx, eventNIDs) @@ -220,8 +207,8 @@ func (r *RoomserverQueryAPI) loadEvents( return result, nil } -// QueryMembershipForUser implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryMembershipForUser( +// QueryMembershipForUser implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryMembershipForUser( ctx context.Context, request *api.QueryMembershipForUserRequest, response *api.QueryMembershipForUserResponse, @@ -251,8 +238,8 @@ func (r *RoomserverQueryAPI) QueryMembershipForUser( return nil } -// QueryMembershipsForRoom implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryMembershipsForRoom( +// QueryMembershipsForRoom implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryMembershipsForRoom( ctx context.Context, request *api.QueryMembershipsForRoomRequest, response *api.QueryMembershipsForRoomResponse, @@ -366,8 +353,8 @@ func getMembershipsAtState( return events, nil } -// QueryInvitesForUser implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryInvitesForUser( +// QueryInvitesForUser implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryInvitesForUser( ctx context.Context, request *api.QueryInvitesForUserRequest, response *api.QueryInvitesForUserResponse, @@ -400,8 +387,8 @@ func (r *RoomserverQueryAPI) QueryInvitesForUser( return nil } -// QueryServerAllowedToSeeEvent implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryServerAllowedToSeeEvent( +// QueryServerAllowedToSeeEvent implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryServerAllowedToSeeEvent( ctx context.Context, request *api.QueryServerAllowedToSeeEventRequest, response *api.QueryServerAllowedToSeeEventResponse, @@ -424,7 +411,7 @@ func (r *RoomserverQueryAPI) QueryServerAllowedToSeeEvent( return } -func (r *RoomserverQueryAPI) checkServerAllowedToSeeEvent( +func (r *RoomserverInternalAPI) checkServerAllowedToSeeEvent( ctx context.Context, eventID string, serverName gomatrixserverlib.ServerName, isServerInRoom bool, ) (bool, error) { roomState := state.NewStateResolution(r.DB) @@ -443,8 +430,8 @@ func (r *RoomserverQueryAPI) checkServerAllowedToSeeEvent( return auth.IsServerAllowed(serverName, isServerInRoom, stateAtEvent), nil } -// QueryMissingEvents implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryMissingEvents( +// QueryMissingEvents implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryMissingEvents( ctx context.Context, request *api.QueryMissingEventsRequest, response *api.QueryMissingEventsResponse, @@ -489,7 +476,7 @@ func (r *RoomserverQueryAPI) QueryMissingEvents( } // QueryBackfill implements api.RoomServerQueryAPI -func (r *RoomserverQueryAPI) QueryBackfill( +func (r *RoomserverInternalAPI) QueryBackfill( ctx context.Context, request *api.QueryBackfillRequest, response *api.QueryBackfillResponse, @@ -542,7 +529,7 @@ func (r *RoomserverQueryAPI) QueryBackfill( return err } -func (r *RoomserverQueryAPI) backfillViaFederation(ctx context.Context, req *api.QueryBackfillRequest, res *api.QueryBackfillResponse) error { +func (r *RoomserverInternalAPI) backfillViaFederation(ctx context.Context, req *api.QueryBackfillRequest, res *api.QueryBackfillResponse) error { roomVer, err := r.DB.GetRoomVersionForRoom(ctx, req.RoomID) if err != nil { return fmt.Errorf("backfillViaFederation: unknown room version for room %s : %w", req.RoomID, err) @@ -600,7 +587,7 @@ func (r *RoomserverQueryAPI) backfillViaFederation(ctx context.Context, req *api return nil } -func (r *RoomserverQueryAPI) isServerCurrentlyInRoom(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID string) (bool, error) { +func (r *RoomserverInternalAPI) isServerCurrentlyInRoom(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID string) (bool, error) { roomNID, err := r.DB.RoomNID(ctx, roomID) if err != nil { return false, err @@ -624,7 +611,7 @@ func (r *RoomserverQueryAPI) isServerCurrentlyInRoom(ctx context.Context, server // fetchAndStoreMissingEvents does a best-effort fetch and store of missing events specified in stateIDs. Returns no error as it is just // best effort. -func (r *RoomserverQueryAPI) fetchAndStoreMissingEvents(ctx context.Context, roomVer gomatrixserverlib.RoomVersion, +func (r *RoomserverInternalAPI) fetchAndStoreMissingEvents(ctx context.Context, roomVer gomatrixserverlib.RoomVersion, backfillRequester *backfillRequester, stateIDs []string) { servers := backfillRequester.servers @@ -684,7 +671,7 @@ func (r *RoomserverQueryAPI) fetchAndStoreMissingEvents(ctx context.Context, roo // TODO: Remove this when we have tests to assert correctness of this function // nolint:gocyclo -func (r *RoomserverQueryAPI) scanEventTree( +func (r *RoomserverInternalAPI) scanEventTree( ctx context.Context, front []string, visited map[string]bool, limit int, serverName gomatrixserverlib.ServerName, ) ([]types.EventNID, error) { @@ -777,8 +764,8 @@ BFSLoop: return resultNIDs, err } -// QueryStateAndAuthChain implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryStateAndAuthChain( +// QueryStateAndAuthChain implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryStateAndAuthChain( ctx context.Context, request *api.QueryStateAndAuthChainRequest, response *api.QueryStateAndAuthChainResponse, @@ -837,7 +824,7 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( return err } -func (r *RoomserverQueryAPI) loadStateAtEventIDs(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) { +func (r *RoomserverInternalAPI) loadStateAtEventIDs(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) { roomState := state.NewStateResolution(r.DB) prevStates, err := r.DB.StateAtEventIDs(ctx, eventIDs) if err != nil { @@ -942,8 +929,8 @@ func persistEvents(ctx context.Context, db storage.Database, events []gomatrixse return roomNID, backfilledEventMap } -// QueryRoomVersionCapabilities implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryRoomVersionCapabilities( +// QueryRoomVersionCapabilities implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryRoomVersionCapabilities( ctx context.Context, request *api.QueryRoomVersionCapabilitiesRequest, response *api.QueryRoomVersionCapabilitiesResponse, @@ -960,8 +947,8 @@ func (r *RoomserverQueryAPI) QueryRoomVersionCapabilities( return nil } -// QueryRoomVersionCapabilities implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryRoomVersionForRoom( +// QueryRoomVersionCapabilities implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryRoomVersionForRoom( ctx context.Context, request *api.QueryRoomVersionForRoomRequest, response *api.QueryRoomVersionForRoomResponse, @@ -979,176 +966,3 @@ func (r *RoomserverQueryAPI) QueryRoomVersionForRoom( r.ImmutableCache.StoreRoomVersion(request.RoomID, response.RoomVersion) return nil } - -// SetupHTTP adds the RoomserverQueryAPI handlers to the http.ServeMux. -// nolint: gocyclo -func (r *RoomserverQueryAPI) SetupHTTP(servMux *http.ServeMux) { - servMux.Handle( - api.RoomserverQueryLatestEventsAndStatePath, - common.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse { - var request api.QueryLatestEventsAndStateRequest - var response api.QueryLatestEventsAndStateResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryLatestEventsAndState(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryStateAfterEventsPath, - common.MakeInternalAPI("queryStateAfterEvents", func(req *http.Request) util.JSONResponse { - var request api.QueryStateAfterEventsRequest - var response api.QueryStateAfterEventsResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryStateAfterEvents(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryEventsByIDPath, - common.MakeInternalAPI("queryEventsByID", func(req *http.Request) util.JSONResponse { - var request api.QueryEventsByIDRequest - var response api.QueryEventsByIDResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryEventsByID(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryMembershipForUserPath, - common.MakeInternalAPI("QueryMembershipForUser", func(req *http.Request) util.JSONResponse { - var request api.QueryMembershipForUserRequest - var response api.QueryMembershipForUserResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryMembershipForUser(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryMembershipsForRoomPath, - common.MakeInternalAPI("queryMembershipsForRoom", func(req *http.Request) util.JSONResponse { - var request api.QueryMembershipsForRoomRequest - var response api.QueryMembershipsForRoomResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryMembershipsForRoom(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryInvitesForUserPath, - common.MakeInternalAPI("queryInvitesForUser", func(req *http.Request) util.JSONResponse { - var request api.QueryInvitesForUserRequest - var response api.QueryInvitesForUserResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryInvitesForUser(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryServerAllowedToSeeEventPath, - common.MakeInternalAPI("queryServerAllowedToSeeEvent", func(req *http.Request) util.JSONResponse { - var request api.QueryServerAllowedToSeeEventRequest - var response api.QueryServerAllowedToSeeEventResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryServerAllowedToSeeEvent(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryMissingEventsPath, - common.MakeInternalAPI("queryMissingEvents", func(req *http.Request) util.JSONResponse { - var request api.QueryMissingEventsRequest - var response api.QueryMissingEventsResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryMissingEvents(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryStateAndAuthChainPath, - common.MakeInternalAPI("queryStateAndAuthChain", func(req *http.Request) util.JSONResponse { - var request api.QueryStateAndAuthChainRequest - var response api.QueryStateAndAuthChainResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryStateAndAuthChain(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryBackfillPath, - common.MakeInternalAPI("QueryBackfill", func(req *http.Request) util.JSONResponse { - var request api.QueryBackfillRequest - var response api.QueryBackfillResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryBackfill(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryRoomVersionCapabilitiesPath, - common.MakeInternalAPI("QueryRoomVersionCapabilities", func(req *http.Request) util.JSONResponse { - var request api.QueryRoomVersionCapabilitiesRequest - var response api.QueryRoomVersionCapabilitiesResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryRoomVersionCapabilities(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryRoomVersionForRoomPath, - common.MakeInternalAPI("QueryRoomVersionForRoom", func(req *http.Request) util.JSONResponse { - var request api.QueryRoomVersionForRoomRequest - var response api.QueryRoomVersionForRoomResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryRoomVersionForRoom(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) -} diff --git a/roomserver/query/backfill.go b/roomserver/internal/query_backfill.go similarity index 99% rename from roomserver/query/backfill.go rename to roomserver/internal/query_backfill.go index f518de3e8..d42038e74 100644 --- a/roomserver/query/backfill.go +++ b/roomserver/internal/query_backfill.go @@ -1,4 +1,4 @@ -package query +package internal import ( "context" diff --git a/roomserver/query/query_test.go b/roomserver/internal/query_test.go similarity index 96% rename from roomserver/query/query_test.go rename to roomserver/internal/query_test.go index 8fb6a082e..211ab5083 100644 --- a/roomserver/query/query_test.go +++ b/roomserver/internal/query_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package query +package internal import ( "context" @@ -24,7 +24,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -// used to implement RoomserverQueryAPIEventDB to test getAuthChain +// used to implement RoomserverInternalAPIEventDB to test getAuthChain type getEventDB struct { eventMap map[string]gomatrixserverlib.Event } @@ -79,7 +79,7 @@ func (db *getEventDB) addFakeEvents(graph map[string][]string) error { return nil } -// EventsFromIDs implements RoomserverQueryAPIEventDB +// EventsFromIDs implements RoomserverInternalAPIEventDB func (db *getEventDB) EventsFromIDs(ctx context.Context, eventIDs []string) (res []types.Event, err error) { for _, evID := range eventIDs { res = append(res, types.Event{ diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index 6fb2caff9..916e25fbd 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -20,12 +20,8 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" - asQuery "github.com/matrix-org/dendrite/appservice/query" - "github.com/matrix-org/dendrite/common/basecomponent" - "github.com/matrix-org/dendrite/roomserver/alias" - "github.com/matrix-org/dendrite/roomserver/input" - "github.com/matrix-org/dendrite/roomserver/query" + "github.com/matrix-org/dendrite/roomserver/internal" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/sirupsen/logrus" ) @@ -35,45 +31,27 @@ import ( // allowing other components running in the same process to hit the query the // APIs directly instead of having to use HTTP. func SetupRoomServerComponent( - base *basecomponent.BaseDendrite, keyRing gomatrixserverlib.JSONVerifier, + base *basecomponent.BaseDendrite, + keyRing gomatrixserverlib.JSONVerifier, fedClient *gomatrixserverlib.FederationClient, -) (api.RoomserverAliasAPI, api.RoomserverInputAPI, api.RoomserverQueryAPI) { +) api.RoomserverInternalAPI { roomserverDB, err := storage.Open(string(base.Cfg.Database.RoomServer)) if err != nil { logrus.WithError(err).Panicf("failed to connect to room server db") } - inputAPI := input.RoomserverInputAPI{ + internalAPI := internal.RoomserverInternalAPI{ DB: roomserverDB, + Cfg: base.Cfg, Producer: base.KafkaProducer, OutputRoomEventTopic: string(base.Cfg.Kafka.Topics.OutputRoomEvent), + ImmutableCache: base.ImmutableCache, + ServerName: base.Cfg.Matrix.ServerName, + FedClient: fedClient, + KeyRing: keyRing, } - inputAPI.SetupHTTP(http.DefaultServeMux) + internalAPI.SetupHTTP(http.DefaultServeMux) - queryAPI := query.RoomserverQueryAPI{ - DB: roomserverDB, - ImmutableCache: base.ImmutableCache, - ServerName: base.Cfg.Matrix.ServerName, - FedClient: fedClient, - // TODO: We should have a key server so we don't keep adding components - // which talk to the same DB. - KeyRing: keyRing, - } - - queryAPI.SetupHTTP(http.DefaultServeMux) - - asAPI := asQuery.AppServiceQueryAPI{Cfg: base.Cfg} - - aliasAPI := alias.RoomserverAliasAPI{ - DB: roomserverDB, - Cfg: base.Cfg, - InputAPI: &inputAPI, - QueryAPI: &queryAPI, - AppserviceAPI: &asAPI, - } - - aliasAPI.SetupHTTP(http.DefaultServeMux) - - return &aliasAPI, &inputAPI, &queryAPI + return &internalAPI } diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 1d512972d..987cc5df6 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -32,10 +32,10 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { - roomServerConsumer *common.ContinualConsumer - db storage.Database - notifier *sync.Notifier - query api.RoomserverQueryAPI + rsAPI api.RoomserverInternalAPI + rsConsumer *common.ContinualConsumer + db storage.Database + notifier *sync.Notifier } // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. @@ -44,7 +44,7 @@ func NewOutputRoomEventConsumer( kafkaConsumer sarama.Consumer, n *sync.Notifier, store storage.Database, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, ) *OutputRoomEventConsumer { consumer := common.ContinualConsumer{ @@ -53,10 +53,10 @@ func NewOutputRoomEventConsumer( PartitionStore: store, } s := &OutputRoomEventConsumer{ - roomServerConsumer: &consumer, - db: store, - notifier: n, - query: queryAPI, + rsConsumer: &consumer, + db: store, + notifier: n, + rsAPI: rsAPI, } consumer.ProcessMessage = s.onMessage @@ -65,7 +65,7 @@ func NewOutputRoomEventConsumer( // Start consuming from room servers func (s *OutputRoomEventConsumer) Start() error { - return s.roomServerConsumer.Start() + return s.rsConsumer.Start() } // onMessage is called when the sync server receives a new event from the room server output log. @@ -226,7 +226,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( // from the roomserver using the query API. eventReq := api.QueryEventsByIDRequest{EventIDs: missing} var eventResp api.QueryEventsByIDResponse - if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { + if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { return nil, err } diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index c48414ab6..5105e224a 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -34,7 +34,7 @@ import ( type messagesReq struct { ctx context.Context db storage.Database - queryAPI api.RoomserverQueryAPI + rsAPI api.RoomserverInternalAPI federation *gomatrixserverlib.FederationClient cfg *config.Dendrite roomID string @@ -59,7 +59,7 @@ const defaultMessagesLimit = 10 func OnIncomingMessagesRequest( req *http.Request, db storage.Database, roomID string, federation *gomatrixserverlib.FederationClient, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, cfg *config.Dendrite, ) util.JSONResponse { var err error @@ -135,7 +135,7 @@ func OnIncomingMessagesRequest( mReq := messagesReq{ ctx: req.Context(), db: db, - queryAPI: queryAPI, + rsAPI: rsAPI, federation: federation, cfg: cfg, roomID: roomID, @@ -360,7 +360,7 @@ func (r *messagesReq) handleNonEmptyEventsSlice(streamEvents []types.StreamEvent // the room or sending the request. func (r *messagesReq) backfill(roomID string, fromEventIDs []string, limit int) ([]gomatrixserverlib.HeaderedEvent, error) { var res api.QueryBackfillResponse - err := r.queryAPI.QueryBackfill(context.Background(), &api.QueryBackfillRequest{ + err := r.rsAPI.QueryBackfill(context.Background(), &api.QueryBackfillRequest{ RoomID: roomID, EarliestEventsIDs: fromEventIDs, Limit: limit, diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index 9078b87ff..5a36a279f 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -40,7 +40,7 @@ const pathPrefixR0 = "/_matrix/client/r0" func Setup( apiMux *mux.Router, srp *sync.RequestPool, syncDB storage.Database, deviceDB devices.Database, federation *gomatrixserverlib.FederationClient, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, cfg *config.Dendrite, ) { r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() @@ -61,6 +61,6 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], federation, queryAPI, cfg) + return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], federation, rsAPI, cfg) })).Methods(http.MethodGet, http.MethodOptions) } diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index 1535d2b13..5ab1ec7c8 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -38,7 +38,7 @@ func SetupSyncAPIComponent( base *basecomponent.BaseDendrite, deviceDB devices.Database, accountsDB accounts.Database, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, federation *gomatrixserverlib.FederationClient, cfg *config.Dendrite, ) { @@ -61,7 +61,7 @@ func SetupSyncAPIComponent( requestPool := sync.NewRequestPool(syncDB, notifier, accountsDB) roomConsumer := consumers.NewOutputRoomEventConsumer( - base.Cfg, base.KafkaConsumer, notifier, syncDB, queryAPI, + base.Cfg, base.KafkaConsumer, notifier, syncDB, rsAPI, ) if err = roomConsumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start room server consumer") @@ -81,5 +81,5 @@ func SetupSyncAPIComponent( logrus.WithError(err).Panicf("failed to start typing server consumer") } - routing.Setup(base.APIMux, requestPool, syncDB, deviceDB, federation, queryAPI, cfg) + routing.Setup(base.APIMux, requestPool, syncDB, deviceDB, federation, rsAPI, cfg) } From b28674435e3024bf0e4723a9cf53180607f2045e Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 1 May 2020 11:01:34 +0100 Subject: [PATCH 12/12] Correctly generate backpagination tokens for events which have the same depth (#996) * Correctly generate backpagination tokens for events which have the same depth With tests. Unfortunately the code around here is hard to understand. There will be a subsequent PR which fixes this up now that we have a test harness in place. * Add postgres impl * More linting * Fix psql statement so it actually works --- syncapi/routing/messages.go | 14 +-- syncapi/storage/interface.go | 6 +- .../postgres/output_room_events_table.go | 27 +++-- .../output_room_events_topology_table.go | 41 ++++--- syncapi/storage/postgres/syncserver.go | 18 ++- .../sqlite3/output_room_events_table.go | 29 +++-- .../output_room_events_topology_table.go | 43 ++++--- syncapi/storage/sqlite3/syncserver.go | 22 ++-- syncapi/storage/storage_test.go | 108 +++++++++++++++++- syncapi/types/types.go | 10 +- 10 files changed, 238 insertions(+), 80 deletions(-) diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 5105e224a..d7e39704d 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -229,14 +229,14 @@ func (r *messagesReq) retrieveEvents() ( // change the way topological positions are defined (as depth isn't the most // reliable way to define it), it would be easier and less troublesome to // only have to change it in one place, i.e. the database. - startPos, err := r.db.EventPositionInTopology( + startPos, startStreamPos, err := r.db.EventPositionInTopology( r.ctx, events[0].EventID(), ) if err != nil { err = fmt.Errorf("EventPositionInTopology: for start event %s: %w", events[0].EventID(), err) return } - endPos, err := r.db.EventPositionInTopology( + endPos, endStreamPos, err := r.db.EventPositionInTopology( r.ctx, events[len(events)-1].EventID(), ) if err != nil { @@ -246,10 +246,10 @@ func (r *messagesReq) retrieveEvents() ( // Generate pagination tokens to send to the client using the positions // retrieved previously. start = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, startPos, 0, + types.PaginationTokenTypeTopology, startPos, startStreamPos, ) end = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, endPos, 0, + types.PaginationTokenTypeTopology, endPos, endStreamPos, ) if r.backwardOrdering { @@ -407,13 +407,13 @@ func setToDefault( // go 1 earlier than the first event so we correctly fetch the earliest event to = types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) } else { - var pos types.StreamPosition - pos, err = db.MaxTopologicalPosition(ctx, roomID) + var pos, stream types.StreamPosition + pos, stream, err = db.MaxTopologicalPosition(ctx, roomID) if err != nil { return } - to = types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, pos, 0) + to = types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, pos, stream) } return diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index bd9504db8..7d6376438 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -91,8 +91,8 @@ type Database interface { // GetEventsInRange retrieves all of the events on a given ordering using the // given extremities and limit. GetEventsInRange(ctx context.Context, from, to *types.PaginationToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error) - // EventPositionInTopology returns the depth of the given event. - EventPositionInTopology(ctx context.Context, eventID string) (types.StreamPosition, error) + // EventPositionInTopology returns the depth and stream position of the given event. + EventPositionInTopology(ctx context.Context, eventID string) (depth types.StreamPosition, stream types.StreamPosition, err error) // EventsAtTopologicalPosition returns all of the events matching a given // position in the topology of a given room. EventsAtTopologicalPosition(ctx context.Context, roomID string, pos types.StreamPosition) ([]types.StreamEvent, error) @@ -100,7 +100,7 @@ type Database interface { // extremities we know of for a given room. BackwardExtremitiesForRoom(ctx context.Context, roomID string) (backwardExtremities []string, err error) // MaxTopologicalPosition returns the highest topological position for a given room. - MaxTopologicalPosition(ctx context.Context, roomID string) (types.StreamPosition, error) + MaxTopologicalPosition(ctx context.Context, roomID string) (depth types.StreamPosition, stream types.StreamPosition, err error) // StreamEventsToEvents converts streamEvent to Event. If device is non-nil and // matches the streamevent.transactionID device then the transaction ID gets // added to the unsigned section of the output event. diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index 0b53dfa9e..769823e92 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -94,6 +94,9 @@ const selectEarlyEventsSQL = "" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id ASC LIMIT $4" +const selectStreamPositionForEventIDSQL = "" + + "SELECT id FROM syncapi_output_room_events WHERE event_id = $1" + const selectMaxEventIDSQL = "" + "SELECT MAX(id) FROM syncapi_output_room_events" @@ -111,13 +114,14 @@ const selectStateInRangeSQL = "" + " LIMIT $8" type outputRoomEventsStatements struct { - insertEventStmt *sql.Stmt - selectEventsStmt *sql.Stmt - selectMaxEventIDStmt *sql.Stmt - selectRecentEventsStmt *sql.Stmt - selectRecentEventsForSyncStmt *sql.Stmt - selectEarlyEventsStmt *sql.Stmt - selectStateInRangeStmt *sql.Stmt + insertEventStmt *sql.Stmt + selectEventsStmt *sql.Stmt + selectMaxEventIDStmt *sql.Stmt + selectRecentEventsStmt *sql.Stmt + selectRecentEventsForSyncStmt *sql.Stmt + selectEarlyEventsStmt *sql.Stmt + selectStateInRangeStmt *sql.Stmt + selectStreamPositionForEventIDStmt *sql.Stmt } func (s *outputRoomEventsStatements) prepare(db *sql.DB) (err error) { @@ -146,9 +150,18 @@ func (s *outputRoomEventsStatements) prepare(db *sql.DB) (err error) { if s.selectStateInRangeStmt, err = db.Prepare(selectStateInRangeSQL); err != nil { return } + if s.selectStreamPositionForEventIDStmt, err = db.Prepare(selectStreamPositionForEventIDSQL); err != nil { + return + } return } +func (s *outputRoomEventsStatements) selectStreamPositionForEventID(ctx context.Context, eventID string) (types.StreamPosition, error) { + var id int64 + err := s.selectStreamPositionForEventIDStmt.QueryRowContext(ctx, eventID).Scan(&id) + return types.StreamPosition(id), err +} + // selectStateInRange returns the state events between the two given PDU stream positions, exclusive of oldPos, inclusive of newPos. // Results are bucketed based on the room ID. If the same state is overwritten multiple times between the // two positions, only the most recent state is returned. diff --git a/syncapi/storage/postgres/output_room_events_topology_table.go b/syncapi/storage/postgres/output_room_events_topology_table.go index 280d4ec39..f77365c8d 100644 --- a/syncapi/storage/postgres/output_room_events_topology_table.go +++ b/syncapi/storage/postgres/output_room_events_topology_table.go @@ -32,35 +32,44 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events_topology ( -- The place of the event in the room's topology. This can usually be determined -- from the event's depth. topological_position BIGINT NOT NULL, + stream_position BIGINT NOT NULL, -- The 'room_id' key for the event. room_id TEXT NOT NULL ); -- The topological order will be used in events selection and ordering -CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_topological_position_idx ON syncapi_output_room_events_topology(topological_position, room_id); +CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_topological_position_idx ON syncapi_output_room_events_topology(topological_position, stream_position, room_id); ` const insertEventInTopologySQL = "" + - "INSERT INTO syncapi_output_room_events_topology (event_id, topological_position, room_id)" + - " VALUES ($1, $2, $3)" + - " ON CONFLICT (topological_position, room_id) DO UPDATE SET event_id = $1" + "INSERT INTO syncapi_output_room_events_topology (event_id, topological_position, room_id, stream_position)" + + " VALUES ($1, $2, $3, $4)" + + " ON CONFLICT (topological_position, stream_position, room_id) DO UPDATE SET event_id = $1" const selectEventIDsInRangeASCSQL = "" + "SELECT event_id FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1 AND topological_position > $2 AND topological_position <= $3" + - " ORDER BY topological_position ASC LIMIT $4" + " WHERE room_id = $1 AND" + + "(topological_position > $2 AND topological_position < $3) OR" + + "(topological_position = $4 AND stream_position <= $5)" + + " ORDER BY topological_position ASC, stream_position ASC LIMIT $6" const selectEventIDsInRangeDESCSQL = "" + "SELECT event_id FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1 AND topological_position > $2 AND topological_position <= $3" + - " ORDER BY topological_position DESC LIMIT $4" + " WHERE room_id = $1 AND" + + "(topological_position > $2 AND topological_position < $3) OR" + + "(topological_position = $4 AND stream_position <= $5)" + + " ORDER BY topological_position DESC, stream_position DESC LIMIT $6" const selectPositionInTopologySQL = "" + "SELECT topological_position FROM syncapi_output_room_events_topology" + " WHERE event_id = $1" + // Select the max topological position for the room, then sort by stream position and take the highest, + // returning both topological and stream positions. const selectMaxPositionInTopologySQL = "" + - "SELECT MAX(topological_position) FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1" + "SELECT topological_position, stream_position FROM syncapi_output_room_events_topology" + + " WHERE topological_position=(" + + "SELECT MAX(topological_position) FROM syncapi_output_room_events_topology WHERE room_id=$1" + + ") ORDER BY stream_position DESC LIMIT 1" const selectEventIDsFromPositionSQL = "" + "SELECT event_id FROM syncapi_output_room_events_topology" + @@ -104,10 +113,10 @@ func (s *outputRoomEventsTopologyStatements) prepare(db *sql.DB) (err error) { // insertEventInTopology inserts the given event in the room's topology, based // on the event's depth. func (s *outputRoomEventsTopologyStatements) insertEventInTopology( - ctx context.Context, event *gomatrixserverlib.HeaderedEvent, + ctx context.Context, event *gomatrixserverlib.HeaderedEvent, pos types.StreamPosition, ) (err error) { _, err = s.insertEventInTopologyStmt.ExecContext( - ctx, event.EventID(), event.Depth(), event.RoomID(), + ctx, event.EventID(), event.Depth(), event.RoomID(), pos, ) return } @@ -116,7 +125,7 @@ func (s *outputRoomEventsTopologyStatements) insertEventInTopology( // given range in a given room's topological order. // Returns an empty slice if no events match the given range. func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( - ctx context.Context, roomID string, fromPos, toPos types.StreamPosition, + ctx context.Context, roomID string, fromPos, toPos, toMicroPos types.StreamPosition, limit int, chronologicalOrder bool, ) (eventIDs []string, err error) { // Decide on the selection's order according to whether chronological order @@ -129,7 +138,7 @@ func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( } // Query the event IDs. - rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, limit) + rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, toPos, toMicroPos, limit) if err == sql.ErrNoRows { // If no event matched the request, return an empty slice. return []string{}, nil @@ -161,8 +170,8 @@ func (s *outputRoomEventsTopologyStatements) selectPositionInTopology( func (s *outputRoomEventsTopologyStatements) selectMaxPositionInTopology( ctx context.Context, roomID string, -) (pos types.StreamPosition, err error) { - err = s.selectMaxPositionInTopologyStmt.QueryRowContext(ctx, roomID).Scan(&pos) +) (pos types.StreamPosition, spos types.StreamPosition, err error) { + err = s.selectMaxPositionInTopologyStmt.QueryRowContext(ctx, roomID).Scan(&pos, &spos) return } diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index ef9707397..744ae7b8d 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -159,7 +159,7 @@ func (d *SyncServerDatasource) WriteEvent( } pduPosition = pos - if err = d.topology.insertEventInTopology(ctx, ev); err != nil { + if err = d.topology.insertEventInTopology(ctx, ev, pos); err != nil { return err } @@ -240,12 +240,13 @@ func (d *SyncServerDatasource) GetEventsInRange( if from.Type == types.PaginationTokenTypeTopology { // Determine the backward and forward limit, i.e. the upper and lower // limits to the selection in the room's topology, from the direction. - var backwardLimit, forwardLimit types.StreamPosition + var backwardLimit, forwardLimit, forwardMicroLimit types.StreamPosition if backwardOrdering { // Backward ordering is antichronological (latest event to oldest // one). backwardLimit = to.PDUPosition forwardLimit = from.PDUPosition + forwardMicroLimit = from.EDUTypingPosition } else { // Forward ordering is chronological (oldest event to latest one). backwardLimit = from.PDUPosition @@ -255,7 +256,7 @@ func (d *SyncServerDatasource) GetEventsInRange( // Select the event IDs from the defined range. var eIDs []string eIDs, err = d.topology.selectEventIDsInRange( - ctx, roomID, backwardLimit, forwardLimit, limit, !backwardOrdering, + ctx, roomID, backwardLimit, forwardLimit, forwardMicroLimit, limit, !backwardOrdering, ) if err != nil { return @@ -301,7 +302,7 @@ func (d *SyncServerDatasource) BackwardExtremitiesForRoom( func (d *SyncServerDatasource) MaxTopologicalPosition( ctx context.Context, roomID string, -) (types.StreamPosition, error) { +) (depth types.StreamPosition, stream types.StreamPosition, err error) { return d.topology.selectMaxPositionInTopology(ctx, roomID) } @@ -318,8 +319,13 @@ func (d *SyncServerDatasource) EventsAtTopologicalPosition( func (d *SyncServerDatasource) EventPositionInTopology( ctx context.Context, eventID string, -) (types.StreamPosition, error) { - return d.topology.selectPositionInTopology(ctx, eventID) +) (depth types.StreamPosition, stream types.StreamPosition, err error) { + depth, err = d.topology.selectPositionInTopology(ctx, eventID) + if err != nil { + return + } + stream, err = d.events.selectStreamPositionForEventID(ctx, eventID) + return } func (d *SyncServerDatasource) SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) { diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 08299f64b..83d7940ad 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -74,6 +74,9 @@ const selectEarlyEventsSQL = "" + const selectMaxEventIDSQL = "" + "SELECT MAX(id) FROM syncapi_output_room_events" +const selectStreamPositionForEventIDSQL = "" + + "SELECT id FROM syncapi_output_room_events WHERE event_id = $1" + // In order for us to apply the state updates correctly, rows need to be ordered in the order they were received (id). /* $1 = oldPos, @@ -99,14 +102,15 @@ const selectStateInRangeSQL = "" + " LIMIT $8" // limit type outputRoomEventsStatements struct { - streamIDStatements *streamIDStatements - insertEventStmt *sql.Stmt - selectEventsStmt *sql.Stmt - selectMaxEventIDStmt *sql.Stmt - selectRecentEventsStmt *sql.Stmt - selectRecentEventsForSyncStmt *sql.Stmt - selectEarlyEventsStmt *sql.Stmt - selectStateInRangeStmt *sql.Stmt + streamIDStatements *streamIDStatements + insertEventStmt *sql.Stmt + selectEventsStmt *sql.Stmt + selectMaxEventIDStmt *sql.Stmt + selectRecentEventsStmt *sql.Stmt + selectRecentEventsForSyncStmt *sql.Stmt + selectEarlyEventsStmt *sql.Stmt + selectStateInRangeStmt *sql.Stmt + selectStreamPositionForEventIDStmt *sql.Stmt } func (s *outputRoomEventsStatements) prepare(db *sql.DB, streamID *streamIDStatements) (err error) { @@ -136,9 +140,18 @@ func (s *outputRoomEventsStatements) prepare(db *sql.DB, streamID *streamIDState if s.selectStateInRangeStmt, err = db.Prepare(selectStateInRangeSQL); err != nil { return } + if s.selectStreamPositionForEventIDStmt, err = db.Prepare(selectStreamPositionForEventIDSQL); err != nil { + return + } return } +func (s *outputRoomEventsStatements) selectStreamPositionForEventID(ctx context.Context, eventID string) (types.StreamPosition, error) { + var id int64 + err := s.selectStreamPositionForEventIDStmt.QueryRowContext(ctx, eventID).Scan(&id) + return types.StreamPosition(id), err +} + // selectStateInRange returns the state events between the two given PDU stream positions, exclusive of oldPos, inclusive of newPos. // Results are bucketed based on the room ID. If the same state is overwritten multiple times between the // two positions, only the most recent state is returned. diff --git a/syncapi/storage/sqlite3/output_room_events_topology_table.go b/syncapi/storage/sqlite3/output_room_events_topology_table.go index a2944c2f9..d64894663 100644 --- a/syncapi/storage/sqlite3/output_room_events_topology_table.go +++ b/syncapi/storage/sqlite3/output_room_events_topology_table.go @@ -27,37 +27,42 @@ const outputRoomEventsTopologySchema = ` -- Stores output room events received from the roomserver. CREATE TABLE IF NOT EXISTS syncapi_output_room_events_topology ( event_id TEXT PRIMARY KEY, - topological_position BIGINT NOT NULL, + topological_position BIGINT NOT NULL, + stream_position BIGINT NOT NULL, room_id TEXT NOT NULL, - UNIQUE(topological_position, room_id) + UNIQUE(topological_position, room_id, stream_position) ); -- The topological order will be used in events selection and ordering --- CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_topological_position_idx ON syncapi_output_room_events_topology(topological_position, room_id); +-- CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_topological_position_idx ON syncapi_output_room_events_topology(topological_position, stream_position, room_id); ` const insertEventInTopologySQL = "" + - "INSERT INTO syncapi_output_room_events_topology (event_id, topological_position, room_id)" + - " VALUES ($1, $2, $3)" + - " ON CONFLICT (topological_position, room_id) DO UPDATE SET event_id = $1" + "INSERT INTO syncapi_output_room_events_topology (event_id, topological_position, room_id, stream_position)" + + " VALUES ($1, $2, $3, $4)" + + " ON CONFLICT DO NOTHING" const selectEventIDsInRangeASCSQL = "" + "SELECT event_id FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1 AND topological_position > $2 AND topological_position <= $3" + - " ORDER BY topological_position ASC LIMIT $4" + " WHERE room_id = $1 AND" + + "(topological_position > $2 AND topological_position < $3) OR" + + "(topological_position = $4 AND stream_position <= $5)" + + " ORDER BY topological_position ASC, stream_position ASC LIMIT $6" const selectEventIDsInRangeDESCSQL = "" + - "SELECT event_id FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1 AND topological_position > $2 AND topological_position <= $3" + - " ORDER BY topological_position DESC LIMIT $4" + "SELECT event_id FROM syncapi_output_room_events_topology" + + " WHERE room_id = $1 AND" + + "(topological_position > $2 AND topological_position < $3) OR" + + "(topological_position = $4 AND stream_position <= $5)" + + " ORDER BY topological_position DESC, stream_position DESC LIMIT $6" const selectPositionInTopologySQL = "" + "SELECT topological_position FROM syncapi_output_room_events_topology" + " WHERE event_id = $1" const selectMaxPositionInTopologySQL = "" + - "SELECT MAX(topological_position) FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1" + "SELECT MAX(topological_position), stream_position FROM syncapi_output_room_events_topology" + + " WHERE room_id = $1 ORDER BY stream_position DESC" const selectEventIDsFromPositionSQL = "" + "SELECT event_id FROM syncapi_output_room_events_topology" + @@ -101,11 +106,11 @@ func (s *outputRoomEventsTopologyStatements) prepare(db *sql.DB) (err error) { // insertEventInTopology inserts the given event in the room's topology, based // on the event's depth. func (s *outputRoomEventsTopologyStatements) insertEventInTopology( - ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, + ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, pos types.StreamPosition, ) (err error) { stmt := common.TxStmt(txn, s.insertEventInTopologyStmt) _, err = stmt.ExecContext( - ctx, event.EventID(), event.Depth(), event.RoomID(), + ctx, event.EventID(), event.Depth(), event.RoomID(), pos, ) return } @@ -115,7 +120,7 @@ func (s *outputRoomEventsTopologyStatements) insertEventInTopology( // Returns an empty slice if no events match the given range. func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( ctx context.Context, txn *sql.Tx, roomID string, - fromPos, toPos types.StreamPosition, + fromPos, toPos, toMicroPos types.StreamPosition, limit int, chronologicalOrder bool, ) (eventIDs []string, err error) { // Decide on the selection's order according to whether chronological order @@ -128,7 +133,7 @@ func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( } // Query the event IDs. - rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, limit) + rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, toPos, toMicroPos, limit) if err == sql.ErrNoRows { // If no event matched the request, return an empty slice. return []string{}, nil @@ -160,9 +165,9 @@ func (s *outputRoomEventsTopologyStatements) selectPositionInTopology( func (s *outputRoomEventsTopologyStatements) selectMaxPositionInTopology( ctx context.Context, txn *sql.Tx, roomID string, -) (pos types.StreamPosition, err error) { +) (pos types.StreamPosition, spos types.StreamPosition, err error) { stmt := common.TxStmt(txn, s.selectMaxPositionInTopologyStmt) - err = stmt.QueryRowContext(ctx, roomID).Scan(&pos) + err = stmt.QueryRowContext(ctx, roomID).Scan(&pos, &spos) return } diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 35774830e..bdf943e08 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -194,7 +194,7 @@ func (d *SyncServerDatasource) WriteEvent( } pduPosition = pos - if err = d.topology.insertEventInTopology(ctx, txn, ev); err != nil { + if err = d.topology.insertEventInTopology(ctx, txn, ev, pos); err != nil { return err } @@ -281,14 +281,16 @@ func (d *SyncServerDatasource) GetEventsInRange( // events must be retrieved from the rooms' topology table rather than the // table contaning the syncapi server's whole stream of events. if from.Type == types.PaginationTokenTypeTopology { + // TODO: ARGH CONFUSING // Determine the backward and forward limit, i.e. the upper and lower // limits to the selection in the room's topology, from the direction. - var backwardLimit, forwardLimit types.StreamPosition + var backwardLimit, forwardLimit, forwardMicroLimit types.StreamPosition if backwardOrdering { // Backward ordering is antichronological (latest event to oldest // one). backwardLimit = to.PDUPosition forwardLimit = from.PDUPosition + forwardMicroLimit = from.EDUTypingPosition } else { // Forward ordering is chronological (oldest event to latest one). backwardLimit = from.PDUPosition @@ -298,7 +300,7 @@ func (d *SyncServerDatasource) GetEventsInRange( // Select the event IDs from the defined range. var eIDs []string eIDs, err = d.topology.selectEventIDsInRange( - ctx, nil, roomID, backwardLimit, forwardLimit, limit, !backwardOrdering, + ctx, nil, roomID, backwardLimit, forwardLimit, forwardMicroLimit, limit, !backwardOrdering, ) if err != nil { return @@ -328,8 +330,7 @@ func (d *SyncServerDatasource) GetEventsInRange( return } } - - return + return events, err } // SyncPosition returns the latest positions for syncing. @@ -353,7 +354,7 @@ func (d *SyncServerDatasource) BackwardExtremitiesForRoom( // room. func (d *SyncServerDatasource) MaxTopologicalPosition( ctx context.Context, roomID string, -) (types.StreamPosition, error) { +) (types.StreamPosition, types.StreamPosition, error) { return d.topology.selectMaxPositionInTopology(ctx, nil, roomID) } @@ -372,8 +373,13 @@ func (d *SyncServerDatasource) EventsAtTopologicalPosition( func (d *SyncServerDatasource) EventPositionInTopology( ctx context.Context, eventID string, -) (types.StreamPosition, error) { - return d.topology.selectPositionInTopology(ctx, nil, eventID) +) (depth types.StreamPosition, stream types.StreamPosition, err error) { + depth, err = d.topology.selectPositionInTopology(ctx, nil, eventID) + if err != nil { + return + } + stream, err = d.events.selectStreamPositionForEventID(ctx, eventID) + return } // SyncStreamPosition returns the latest position in the sync stream. Returns 0 if there are no events yet. diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index e591e7ed7..a57d59176 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -182,7 +182,7 @@ func TestSyncResponse(t *testing.T) { // limit set to 5 return db.CompleteSync(ctx, testUserIDA, 5) }, - // want the last 5 events, NOT the last 10. + // want the last 5 events WantTimeline: events[len(events)-5:], // want all state for the room WantState: state, @@ -248,11 +248,11 @@ func TestGetEventsInRangeWithTopologyToken(t *testing.T) { db := MustCreateDatabase(t) events, _ := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) MustWriteEvents(t, db, events) - latest, err := db.MaxTopologicalPosition(ctx, testRoomID) + latest, latestStream, err := db.MaxTopologicalPosition(ctx, testRoomID) if err != nil { t.Fatalf("failed to get MaxTopologicalPosition: %s", err) } - from := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, latest, 0) + from := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, latest, latestStream) // head towards the beginning of time to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) @@ -265,6 +265,105 @@ func TestGetEventsInRangeWithTopologyToken(t *testing.T) { assertEventsEqual(t, "", true, gots, reversed(events[len(events)-5:])) } +// The purpose of this test is to make sure that backpagination returns all events, even if some events have the same depth. +// For cases where events have the same depth, the streaming token should be used to tie break so events written via WriteEvent +// will appear FIRST when going backwards. This test creates a DAG like: +// .-----> Message ---. +// Create -> Membership --------> Message -------> Message +// `-----> Message ---` +// depth 1 2 3 4 +// +// With a total depth of 4. It tests that: +// - Backpagination over the whole fork should include all messages and not leave any out. +// - Backpagination from the middle of the fork should not return duplicates (things later than the token). +func TestGetEventsInRangeWithEventsSameDepth(t *testing.T) { + t.Parallel() + db := MustCreateDatabase(t) + + var events []gomatrixserverlib.HeaderedEvent + events = append(events, MustCreateEvent(t, testRoomID, nil, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"room_version":"4","creator":"%s"}`, testUserIDA)), + Type: "m.room.create", + StateKey: &emptyStateKey, + Sender: testUserIDA, + Depth: int64(len(events) + 1), + })) + events = append(events, MustCreateEvent(t, testRoomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"membership":"join"}`)), + Type: "m.room.member", + StateKey: &testUserIDA, + Sender: testUserIDA, + Depth: int64(len(events) + 1), + })) + // fork the dag into three, same prev_events and depth + parent := []gomatrixserverlib.HeaderedEvent{events[len(events)-1]} + depth := int64(len(events) + 1) + for i := 0; i < 3; i++ { + events = append(events, MustCreateEvent(t, testRoomID, parent, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"body":"Message A %d"}`, i+1)), + Type: "m.room.message", + Sender: testUserIDA, + Depth: depth, + })) + } + // merge the fork, prev_events are all 3 messages, depth is increased by 1. + events = append(events, MustCreateEvent(t, testRoomID, events[len(events)-3:], &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"body":"Message merge"}`)), + Type: "m.room.message", + Sender: testUserIDA, + Depth: depth + 1, + })) + MustWriteEvents(t, db, events) + latestPos, latestStreamPos, err := db.EventPositionInTopology(ctx, events[len(events)-1].EventID()) + if err != nil { + t.Fatalf("failed to get EventPositionInTopology: %s", err) + } + topoPos, streamPos, err := db.EventPositionInTopology(ctx, events[len(events)-3].EventID()) // Message 2 + if err != nil { + t.Fatalf("failed to get EventPositionInTopology for event: %s", err) + } + fromLatest := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, latestPos, latestStreamPos) + fromFork := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, topoPos, streamPos) + // head towards the beginning of time + to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) + + testCases := []struct { + Name string + From *types.PaginationToken + Limit int + Wants []gomatrixserverlib.HeaderedEvent + }{ + { + Name: "Pagination over the whole fork", + From: fromLatest, + Limit: 5, + Wants: reversed(events[len(events)-5:]), + }, + { + Name: "Paginating to the middle of the fork", + From: fromLatest, + Limit: 2, + Wants: reversed(events[len(events)-2:]), + }, + { + Name: "Pagination FROM the middle of the fork", + From: fromFork, + Limit: 3, + Wants: reversed(events[len(events)-5 : len(events)-2]), + }, + } + + for _, tc := range testCases { + // backpaginate messages starting at the latest position. + paginatedEvents, err := db.GetEventsInRange(ctx, tc.From, to, testRoomID, tc.Limit, true) + if err != nil { + t.Fatalf("%s GetEventsInRange returned an error: %s", tc.Name, err) + } + gots := gomatrixserverlib.HeaderedToClientEvents(db.StreamEventsToEvents(&testUserDeviceA, paginatedEvents), gomatrixserverlib.FormatAll) + assertEventsEqual(t, tc.Name, true, gots, tc.Wants) + } +} + func assertEventsEqual(t *testing.T, msg string, checkRoomID bool, gots []gomatrixserverlib.ClientEvent, wants []gomatrixserverlib.HeaderedEvent) { if len(gots) != len(wants) { t.Fatalf("%s response returned %d events, want %d", msg, len(gots), len(wants)) @@ -294,7 +393,8 @@ func assertEventsEqual(t *testing.T, msg string, checkRoomID bool, gots []gomatr t.Errorf("%s event[%d] unsigned mismatch: got %s want %s", msg, i, string(g.Unsigned), string(w.Unsigned())) } if (g.StateKey == nil && w.StateKey() != nil) || (g.StateKey != nil && w.StateKey() == nil) { - t.Fatalf("%s event[%d] state_key [not] missing: got %v want %v", msg, i, g.StateKey, w.StateKey()) + t.Errorf("%s event[%d] state_key [not] missing: got %v want %v", msg, i, g.StateKey, w.StateKey()) + continue } if g.StateKey != nil { if !w.StateKeyEquals(*g.StateKey) { diff --git a/syncapi/types/types.go b/syncapi/types/types.go index cfd49ff1f..c04fe5219 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -64,8 +64,14 @@ const ( // /sync or /messages, for example. type PaginationToken struct { //Position StreamPosition - Type PaginationTokenType - PDUPosition StreamPosition + Type PaginationTokenType + // For /sync, this is the PDU position. For /messages, this is the topological position (depth). + // TODO: Given how different the positions are depending on the token type, they should probably be renamed + // or use different structs altogether. + PDUPosition StreamPosition + // For /sync, this is the EDU position. For /messages, this is the stream (PDU) position. + // TODO: Given how different the positions are depending on the token type, they should probably be renamed + // or use different structs altogether. EDUTypingPosition StreamPosition }