diff --git a/CHANGES.md b/CHANGES.md index 4df8e869a..ee608194d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,39 @@ # Changelog +## Dendrite 0.6.4 (2022-02-21) + +### Features + +* All Client-Server API endpoints are now available under the `/v3` namespace +* The `/whoami` response format now matches the latest Matrix spec version +* Support added for the `/context` endpoint, which should help clients to render quote-replies correctly +* Accounts now have an optional account type field, allowing admin accounts to be created +* Server notices are now supported +* Refactored the user API storage to deduplicate a significant amount of code, as well as merging both user API databases into a single database + * The account database is now used for all user API storage and the device database is now obsolete + * For some installations that have separate account and device databases, this may result in access tokens being revoked and client sessions being logged out — users may need to log in again + * The above can be avoided by moving the `device_devices` table into the account database manually +* Guest registration can now be separately disabled with the new `client_api.guests_disabled` configuration option +* Outbound connections now obey proxy settings from the environment, deprecating the `federation_api.proxy_outbound` configuration options + +### Fixes + +* The roomserver input API will now strictly consume only one database transaction per room, which should prevent situations where the roomserver can deadlock waiting for database connections to become available +* Room joins will now fall back to federation if the local room state is insufficient to create a membership event +* Create events are now correctly filtered from federation `/send` transactions +* Excessive logging when federation is disabled should now be fixed +* Dendrite will no longer panic if trying to retire an invite event that has not been seen yet +* The device list updater will now wait for longer after a connection issue, rather than flooding the logs with errors +* The device list updater will no longer produce unnecessary output events for federated key updates with no changes, which should help to reduce CPU usage +* Local device name changes will now generate key change events correctly +* The sync API will now try to share device list update notifications even if all state key NIDs cannot be fetched +* An off-by-one error in the sync stream token handling which could result in a crash has been fixed +* State events will no longer be re-sent unnecessary by the roomserver to other components if they have already been sent, which should help to reduce the NATS message sizes on the roomserver output topic in some cases +* The roomserver input API now uses the process context and should handle graceful shutdowns better +* Guest registration is now correctly disabled when the `client_api.registration_disabled` configuration option is set +* One-time encryption keys are now cleaned up correctly when a device is logged out or removed +* Invalid state snapshots in the state storage refactoring migration are now reset rather than causing a panic at startup + ## Dendrite 0.6.3 (2022-02-10) ### Features diff --git a/clientapi/routing/state.go b/clientapi/routing/state.go index 088e412c6..d25ee8237 100644 --- a/clientapi/routing/state.go +++ b/clientapi/routing/state.go @@ -235,7 +235,7 @@ func OnIncomingStateTypeRequest( } // If the user has never been in the room then stop at this point. // We won't tell the user about a room they have never joined. - if !membershipRes.HasBeenInRoom { + if !membershipRes.HasBeenInRoom || membershipRes.Membership == gomatrixserverlib.Ban { return util.JSONResponse{ Code: http.StatusForbidden, JSON: jsonerror.Forbidden(fmt.Sprintf("Unknown room %q or user %q has never joined this room", roomID, device.UserID)), diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index de1b2c22e..b7e30ba2e 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -21,8 +21,11 @@ import ( "fmt" "net" "net/http" + "os" "time" + "github.com/matrix-org/gomatrixserverlib" + "github.com/gorilla/mux" "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/embed" @@ -42,8 +45,6 @@ import ( "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/mscs" "github.com/matrix-org/dendrite/userapi" - "github.com/matrix-org/gomatrixserverlib" - "github.com/sirupsen/logrus" _ "github.com/mattn/go-sqlite3" @@ -63,33 +64,42 @@ func main() { if err != nil { panic(err) } - /* - ygg.SetMulticastEnabled(true) - if instancePeer != nil && *instancePeer != "" { - if err = ygg.SetStaticPeer(*instancePeer); err != nil { - logrus.WithError(err).Error("Failed to set static peer") - } + + // iterate through the cli args and check if the config flag was set + configFlagSet := false + for _, arg := range os.Args { + if arg == "--config" || arg == "-config" { + configFlagSet = true + break } - */ + } cfg := &config.Dendrite{} - cfg.Defaults(true) + + // use custom config if config flag is set + if configFlagSet { + cfg = setup.ParseFlags(true) + } else { + cfg.Defaults(true) + cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName)) + cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName)) + cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName)) + cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName)) + cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName)) + cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName)) + cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName)) + cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) + cfg.MSCs.MSCs = []string{"msc2836"} + cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName)) + if err = cfg.Derive(); err != nil { + panic(err) + } + } + + // always override ServerName, PrivateKey and KeyID cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName()) cfg.Global.PrivateKey = ygg.PrivateKey() - cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID) - cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName)) - cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName)) - cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName)) - cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName)) - cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName)) - cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName)) - cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName)) - cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) - cfg.MSCs.MSCs = []string{"msc2836"} - cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName)) - if err = cfg.Derive(); err != nil { - panic(err) - } + cfg.Global.KeyID = signing.KeyID base := base.NewBaseDendrite(cfg, "Monolith") defer base.Close() // nolint: errcheck diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go index c51ecf146..b888b3654 100644 --- a/federationapi/internal/perform.go +++ b/federationapi/internal/perform.go @@ -554,7 +554,6 @@ func (r *FederationInternalAPI) PerformInvite( if err != nil { return fmt.Errorf("r.federation.SendInviteV2: failed to send invite: %w", err) } - logrus.Infof("GOT INVITE RESPONSE %s", string(inviteRes.Event)) inviteEvent, err := inviteRes.Event.UntrustedEvent(request.RoomVersion) if err != nil { diff --git a/internal/version.go b/internal/version.go index a07f01b61..2ea1c5201 100644 --- a/internal/version.go +++ b/internal/version.go @@ -17,7 +17,7 @@ var build string const ( VersionMajor = 0 VersionMinor = 6 - VersionPatch = 3 + VersionPatch = 4 VersionTag = "" // example: "rc1" ) diff --git a/keyserver/api/api.go b/keyserver/api/api.go index 3933961c1..54eb04f8a 100644 --- a/keyserver/api/api.go +++ b/keyserver/api/api.go @@ -15,6 +15,7 @@ package api import ( + "bytes" "context" "encoding/json" "strings" @@ -73,6 +74,26 @@ type DeviceMessage struct { DeviceChangeID int64 } +// DeviceKeysEqual returns true if the device keys updates contain the +// same display name and key JSON. This will return false if either of +// the updates is not a device keys update, or if the user ID/device ID +// differ between the two. +func (m1 *DeviceMessage) DeviceKeysEqual(m2 *DeviceMessage) bool { + if m1.DeviceKeys == nil || m2.DeviceKeys == nil { + return false + } + if m1.UserID != m2.UserID || m1.DeviceID != m2.DeviceID { + return false + } + if m1.DisplayName != m2.DisplayName { + return false // different display names + } + if len(m1.KeyJSON) == 0 || len(m2.KeyJSON) == 0 { + return false // either is empty + } + return bytes.Equal(m1.KeyJSON, m2.KeyJSON) +} + // DeviceKeys represents a set of device keys for a single device // https://matrix.org/docs/spec/client_server/r0.6.1#post-matrix-client-r0-keys-upload type DeviceKeys struct { diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go index c5a5d40c7..974d0196b 100644 --- a/keyserver/internal/device_list_update.go +++ b/keyserver/internal/device_list_update.go @@ -224,7 +224,7 @@ func (u *DeviceListUpdater) update(ctx context.Context, event gomatrixserverlib. }).Info("DeviceListUpdater.Update") // if we haven't missed anything update the database and notify users - if exists { + if exists || event.Deleted { k := event.Keys if event.Deleted { k = nil @@ -241,14 +241,33 @@ func (u *DeviceListUpdater) update(ctx context.Context, event gomatrixserverlib. StreamID: event.StreamID, }, } + + // DeviceKeysJSON will side-effect modify this, so it needs + // to be a copy, not sharing any pointers with the above. + deviceKeysCopy := *keys[0].DeviceKeys + deviceKeysCopy.KeyJSON = nil + existingKeys := []api.DeviceMessage{ + { + Type: keys[0].Type, + DeviceKeys: &deviceKeysCopy, + StreamID: keys[0].StreamID, + }, + } + + // fetch what keys we had already and only emit changes + if err = u.db.DeviceKeysJSON(ctx, existingKeys); err != nil { + // non-fatal, log and continue + util.GetLogger(ctx).WithError(err).WithField("user_id", event.UserID).Errorf( + "failed to query device keys json for calculating diffs", + ) + } + err = u.db.StoreRemoteDeviceKeys(ctx, keys, nil) if err != nil { return false, fmt.Errorf("failed to store remote device keys for %s (%s): %w", event.UserID, event.DeviceID, err) } - // ALWAYS emit key changes when we've been poked over federation even if there's no change - // just in case this poke is important for something. - err = u.producer.ProduceKeyChanges(keys) - if err != nil { + + if err = emitDeviceKeyChanges(u.producer, existingKeys, keys, false); err != nil { return false, fmt.Errorf("failed to produce device key changes for %s (%s): %w", event.UserID, event.DeviceID, err) } return false, nil @@ -454,7 +473,7 @@ func (u *DeviceListUpdater) updateDeviceList(res *gomatrixserverlib.RespUserDevi if err != nil { return fmt.Errorf("failed to mark device list as fresh: %w", err) } - err = emitDeviceKeyChanges(u.producer, existingKeys, keys) + err = emitDeviceKeyChanges(u.producer, existingKeys, keys, false) if err != nil { return fmt.Errorf("failed to emit key changes for fresh device list: %w", err) } diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index 0c264b718..0a8bef95d 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -648,7 +648,7 @@ func (a *KeyInternalAPI) uploadLocalDeviceKeys(ctx context.Context, req *api.Per } return } - err = emitDeviceKeyChanges(a.Producer, existingKeys, keysToStore) + err = emitDeviceKeyChanges(a.Producer, existingKeys, keysToStore, req.OnlyDisplayNameUpdates) if err != nil { util.GetLogger(ctx).Errorf("Failed to emitDeviceKeyChanges: %s", err) } @@ -710,7 +710,11 @@ func (a *KeyInternalAPI) uploadOneTimeKeys(ctx context.Context, req *api.Perform } -func emitDeviceKeyChanges(producer KeyChangeProducer, existing, new []api.DeviceMessage) error { +func emitDeviceKeyChanges(producer KeyChangeProducer, existing, new []api.DeviceMessage, onlyUpdateDisplayName bool) error { + // if we only want to update the display names, we can skip the checks below + if onlyUpdateDisplayName { + return producer.ProduceKeyChanges(new) + } // find keys in new that are not in existing var keysAdded []api.DeviceMessage for _, newKey := range new { @@ -718,7 +722,7 @@ func emitDeviceKeyChanges(producer KeyChangeProducer, existing, new []api.Device for _, existingKey := range existing { // Do not treat the absence of keys as equal, or else we will not emit key changes // when users delete devices which never had a key to begin with as both KeyJSONs are nil. - if bytes.Equal(existingKey.KeyJSON, newKey.KeyJSON) && len(existingKey.KeyJSON) > 0 { + if existingKey.DeviceKeysEqual(&newKey) { exists = true break } diff --git a/roomserver/internal/perform/perform_join.go b/roomserver/internal/perform/perform_join.go index 9d2a66d4c..a40f66d21 100644 --- a/roomserver/internal/perform/perform_join.go +++ b/roomserver/internal/perform/perform_join.go @@ -29,6 +29,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/internal/input" "github.com/matrix-org/dendrite/roomserver/internal/query" "github.com/matrix-org/dendrite/roomserver/storage" + "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" @@ -367,7 +368,15 @@ func buildEvent( StateToFetch: eventsNeeded.Tuples(), }, &queryRes) if err != nil { - return nil, nil, fmt.Errorf("QueryLatestEventsAndState: %w", err) + switch err.(type) { + case types.MissingStateError: + // We know something about the room but the state seems to be + // insufficient to actually build a new event, so in effect we + // had might as well treat the room as if it doesn't exist. + return nil, nil, eventutil.ErrRoomNoExists + default: + return nil, nil, fmt.Errorf("QueryLatestEventsAndState: %w", err) + } } ev, err := eventutil.BuildEvent(ctx, builder, cfg, time.Now(), &eventsNeeded, &queryRes) diff --git a/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go b/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go index 06740dc8b..06442a4c3 100644 --- a/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go +++ b/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go @@ -256,23 +256,17 @@ func UpStateBlocksRefactor(tx *sql.Tx) error { return fmt.Errorf("assertion query failed: %s", err) } if count > 0 { - var debugEventID, debugRoomID string - var debugEventTypeNID, debugStateKeyNID, debugSnapNID, debugDepth int64 - err = tx.QueryRow( - `SELECT event_id, event_type_nid, event_state_key_nid, roomserver_events.state_snapshot_nid, depth, room_id FROM roomserver_events - JOIN roomserver_rooms ON roomserver_rooms.room_nid = roomserver_events.room_nid WHERE roomserver_events.state_snapshot_nid < $1 AND roomserver_events.state_snapshot_nid != 0`, maxsnapshotid, - ).Scan(&debugEventID, &debugEventTypeNID, &debugStateKeyNID, &debugSnapNID, &debugDepth, &debugRoomID) - if err != nil { - logrus.Errorf("cannot extract debug info: %v", err) - } else { - logrus.Errorf( - "Affected row: event_id=%v room_id=%v type=%v state_key=%v snapshot=%v depth=%v", - debugEventID, debugRoomID, debugEventTypeNID, debugStateKeyNID, debugSnapNID, debugDepth, - ) - logrus.Errorf("To fix this manually, run this query first then retry the migration: "+ - "UPDATE roomserver_events SET state_snapshot_nid=0 WHERE event_id='%v'", debugEventID) + var res sql.Result + var c int64 + res, err = tx.Exec(`UPDATE roomserver_events SET state_snapshot_nid = 0 WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, maxsnapshotid) + if err != nil && err != sql.ErrNoRows { + return fmt.Errorf("failed to reset invalid state snapshots: %w", err) + } + if c, err = res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get row count for invalid state snapshots updated: %w", err) + } else if c != count { + return fmt.Errorf("expected to reset %d event(s) but only updated %d event(s)", count, c) } - return fmt.Errorf("%d events exist in roomserver_events which have not been converted to a new state_snapshot_nid; this is a bug, please report", count) } if err = tx.QueryRow(`SELECT COUNT(*) FROM roomserver_rooms WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, maxsnapshotid).Scan(&count); err != nil { return fmt.Errorf("assertion query failed: %s", err) diff --git a/roomserver/storage/postgres/state_snapshot_table.go b/roomserver/storage/postgres/state_snapshot_table.go index ce9f24636..8ed886030 100644 --- a/roomserver/storage/postgres/state_snapshot_table.go +++ b/roomserver/storage/postgres/state_snapshot_table.go @@ -134,7 +134,7 @@ func (s *stateSnapshotStatements) BulkSelectStateBlockNIDs( return nil, err } if i != len(stateNIDs) { - return nil, fmt.Errorf("storage: state NIDs missing from the database (%d != %d)", i, len(stateNIDs)) + return nil, types.MissingStateError(fmt.Sprintf("storage: state NIDs missing from the database (%d != %d)", i, len(stateNIDs))) } return results, nil } diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index b255cfb3f..e270e121c 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -13,6 +13,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" + "github.com/sirupsen/logrus" "github.com/tidwall/gjson" ) @@ -1101,7 +1102,7 @@ func (d *Database) JoinedUsersSetInRooms(ctx context.Context, roomIDs []string) return nil, err } if len(nidToUserID) != len(userNIDToCount) { - return nil, fmt.Errorf("found %d users but only have state key nids for %d of them", len(userNIDToCount), len(nidToUserID)) + logrus.Warnf("SelectJoinedUsersSetForRooms found %d users but BulkSelectEventStateKey only returned state key NIDs for %d of them", len(userNIDToCount), len(nidToUserID)) } result := make(map[string]int, len(userNIDToCount)) for nid, count := range userNIDToCount { diff --git a/roomserver/storage/sqlite3/deltas/2021041615092700_state_blocks_refactor.go b/roomserver/storage/sqlite3/deltas/2021041615092700_state_blocks_refactor.go index 8d0331748..8f5ab8fc5 100644 --- a/roomserver/storage/sqlite3/deltas/2021041615092700_state_blocks_refactor.go +++ b/roomserver/storage/sqlite3/deltas/2021041615092700_state_blocks_refactor.go @@ -179,7 +179,17 @@ func UpStateBlocksRefactor(tx *sql.Tx) error { return fmt.Errorf("assertion query failed: %s", err) } if count > 0 { - return fmt.Errorf("%d events exist in roomserver_events which have not been converted to a new state_snapshot_nid; this is a bug, please report", count) + var res sql.Result + var c int64 + res, err = tx.Exec(`UPDATE roomserver_events SET state_snapshot_nid = 0 WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, oldMaxSnapshotID) + if err != nil && err != sql.ErrNoRows { + return fmt.Errorf("failed to reset invalid state snapshots: %w", err) + } + if c, err = res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get row count for invalid state snapshots updated: %w", err) + } else if c != count { + return fmt.Errorf("expected to reset %d event(s) but only updated %d event(s)", count, c) + } } if err = tx.QueryRow(`SELECT COUNT(*) FROM roomserver_rooms WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, oldMaxSnapshotID).Scan(&count); err != nil { return fmt.Errorf("assertion query failed: %s", err) diff --git a/roomserver/storage/sqlite3/state_snapshot_table.go b/roomserver/storage/sqlite3/state_snapshot_table.go index 3c4bde3f5..01df31e90 100644 --- a/roomserver/storage/sqlite3/state_snapshot_table.go +++ b/roomserver/storage/sqlite3/state_snapshot_table.go @@ -137,7 +137,7 @@ func (s *stateSnapshotStatements) BulkSelectStateBlockNIDs( } } if i != len(stateNIDs) { - return nil, fmt.Errorf("storage: state NIDs missing from the database (%d != %d)", i, len(stateNIDs)) + return nil, types.MissingStateError(fmt.Sprintf("storage: state NIDs missing from the database (%d != %d)", i, len(stateNIDs))) } return results, nil } diff --git a/roomserver/types/types.go b/roomserver/types/types.go index 5d52ccfcd..65fbee04e 100644 --- a/roomserver/types/types.go +++ b/roomserver/types/types.go @@ -213,6 +213,12 @@ type MissingEventError string func (e MissingEventError) Error() string { return string(e) } +// A MissingStateError is an error that happened because the roomserver was +// missing requested state snapshots from its databases. +type MissingStateError string + +func (e MissingStateError) Error() string { return string(e) } + // A RejectedError is returned when an event is stored as rejected. The error // contains the reason why. type RejectedError string diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go new file mode 100644 index 000000000..ef7efa2b0 --- /dev/null +++ b/syncapi/routing/context.go @@ -0,0 +1,191 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "database/sql" + "encoding/json" + "net/http" + "strconv" + + "github.com/matrix-org/dendrite/clientapi/jsonerror" + roomserver "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/syncapi/storage" + userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/sirupsen/logrus" +) + +type ContextRespsonse struct { + End string `json:"end"` + Event gomatrixserverlib.ClientEvent `json:"event"` + EventsAfter []gomatrixserverlib.ClientEvent `json:"events_after,omitempty"` + EventsBefore []gomatrixserverlib.ClientEvent `json:"events_before,omitempty"` + Start string `json:"start"` + State []gomatrixserverlib.ClientEvent `json:"state"` +} + +func Context( + req *http.Request, device *userapi.Device, + rsAPI roomserver.RoomserverInternalAPI, + syncDB storage.Database, + roomID, eventID string, +) util.JSONResponse { + filter, err := parseContextParams(req) + if err != nil { + errMsg := "" + switch err.(type) { + case *json.InvalidUnmarshalError: + errMsg = "unable to parse filter" + case *strconv.NumError: + errMsg = "unable to parse limit" + } + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.InvalidParam(errMsg), + Headers: nil, + } + } + filter.Rooms = append(filter.Rooms, roomID) + + ctx := req.Context() + membershipRes := roomserver.QueryMembershipForUserResponse{} + membershipReq := roomserver.QueryMembershipForUserRequest{UserID: device.UserID, RoomID: roomID} + if err = rsAPI.QueryMembershipForUser(ctx, &membershipReq, &membershipRes); err != nil { + logrus.WithError(err).Error("unable to fo membership") + return jsonerror.InternalServerError() + } + + stateFilter := gomatrixserverlib.StateFilter{ + Limit: 100, + NotSenders: filter.NotSenders, + NotTypes: filter.NotTypes, + Senders: filter.Senders, + Types: filter.Types, + LazyLoadMembers: filter.LazyLoadMembers, + IncludeRedundantMembers: filter.IncludeRedundantMembers, + NotRooms: filter.NotRooms, + Rooms: filter.Rooms, + ContainsURL: filter.ContainsURL, + } + + // TODO: Get the actual state at the last event returned by SelectContextAfterEvent + state, _ := syncDB.CurrentState(ctx, roomID, &stateFilter, nil) + // verify the user is allowed to see the context for this room/event + for _, x := range state { + var hisVis string + hisVis, err = x.HistoryVisibility() + if err != nil { + continue + } + allowed := hisVis == gomatrixserverlib.WorldReadable || membershipRes.Membership == gomatrixserverlib.Join + if !allowed { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("User is not allowed to query context"), + } + } + } + + id, requestedEvent, err := syncDB.SelectContextEvent(ctx, roomID, eventID) + if err != nil { + logrus.WithError(err).WithField("eventID", eventID).Error("unable to find requested event") + return jsonerror.InternalServerError() + } + + eventsBefore, err := syncDB.SelectContextBeforeEvent(ctx, id, roomID, filter) + if err != nil && err != sql.ErrNoRows { + logrus.WithError(err).Error("unable to fetch before events") + return jsonerror.InternalServerError() + } + + _, eventsAfter, err := syncDB.SelectContextAfterEvent(ctx, id, roomID, filter) + if err != nil && err != sql.ErrNoRows { + logrus.WithError(err).Error("unable to fetch after events") + return jsonerror.InternalServerError() + } + + eventsBeforeClient := gomatrixserverlib.HeaderedToClientEvents(eventsBefore, gomatrixserverlib.FormatAll) + eventsAfterClient := gomatrixserverlib.HeaderedToClientEvents(eventsAfter, gomatrixserverlib.FormatAll) + newState := applyLazyLoadMembers(filter, eventsAfterClient, eventsBeforeClient, state) + + response := ContextRespsonse{ + Event: gomatrixserverlib.HeaderedToClientEvent(&requestedEvent, gomatrixserverlib.FormatAll), + EventsAfter: eventsAfterClient, + EventsBefore: eventsBeforeClient, + State: gomatrixserverlib.HeaderedToClientEvents(newState, gomatrixserverlib.FormatAll), + } + + if len(response.State) > filter.Limit { + response.State = response.State[len(response.State)-filter.Limit:] + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: response, + } +} + +func applyLazyLoadMembers(filter *gomatrixserverlib.RoomEventFilter, eventsAfter, eventsBefore []gomatrixserverlib.ClientEvent, state []*gomatrixserverlib.HeaderedEvent) []*gomatrixserverlib.HeaderedEvent { + if filter == nil || !filter.LazyLoadMembers { + return state + } + allEvents := append(eventsBefore, eventsAfter...) + x := make(map[string]bool) + // get members who actually send an event + for _, e := range allEvents { + x[e.Sender] = true + } + + newState := []*gomatrixserverlib.HeaderedEvent{} + for _, event := range state { + if event.Type() != gomatrixserverlib.MRoomMember { + newState = append(newState, event) + } else { + // did the user send an event? + if x[event.Sender()] { + newState = append(newState, event) + } + } + } + return newState +} + +func parseContextParams(req *http.Request) (*gomatrixserverlib.RoomEventFilter, error) { + // Default room filter + filter := &gomatrixserverlib.RoomEventFilter{Limit: 10} + + l := req.URL.Query().Get("limit") + f := req.URL.Query().Get("filter") + if l != "" { + limit, err := strconv.Atoi(l) + if err != nil { + return nil, err + } + // NOTSPEC: feels like a good idea to have an upper bound limit + if limit > 100 { + limit = 100 + } + filter.Limit = limit + } + if f != "" { + if err := json.Unmarshal([]byte(f), &filter); err != nil { + return nil, err + } + } + + return filter, nil +} diff --git a/syncapi/routing/context_test.go b/syncapi/routing/context_test.go new file mode 100644 index 000000000..1b430d83a --- /dev/null +++ b/syncapi/routing/context_test.go @@ -0,0 +1,68 @@ +package routing + +import ( + "net/http" + "reflect" + "testing" + + "github.com/matrix-org/gomatrixserverlib" +) + +func Test_parseContextParams(t *testing.T) { + + noParamsReq, _ := http.NewRequest("GET", "https://localhost:8800/_matrix/client/r0/rooms/!hyi4UaxS9mUXpSG9:localhost:8800/context/%24um_T82QqAXN8PayGiBW7j9WExpqTIQ7-JRq-Q6xpIf8?access_token=5dMB0z4tiulyBvCaIKgyjuWG71ybDiYIwNJVJ2UmxRI", nil) + limit2Req, _ := http.NewRequest("GET", "https://localhost:8800/_matrix/client/r0/rooms/!hyi4UaxS9mUXpSG9:localhost:8800/context/%24um_T82QqAXN8PayGiBW7j9WExpqTIQ7-JRq-Q6xpIf8?access_token=5dMB0z4tiulyBvCaIKgyjuWG71ybDiYIwNJVJ2UmxRI&limit=2", nil) + limit10000Req, _ := http.NewRequest("GET", "https://localhost:8800/_matrix/client/r0/rooms/!hyi4UaxS9mUXpSG9:localhost:8800/context/%24um_T82QqAXN8PayGiBW7j9WExpqTIQ7-JRq-Q6xpIf8?access_token=5dMB0z4tiulyBvCaIKgyjuWG71ybDiYIwNJVJ2UmxRI&limit=10000", nil) + invalidLimitReq, _ := http.NewRequest("GET", "https://localhost:8800/_matrix/client/r0/rooms/!hyi4UaxS9mUXpSG9:localhost:8800/context/%24um_T82QqAXN8PayGiBW7j9WExpqTIQ7-JRq-Q6xpIf8?access_token=5dMB0z4tiulyBvCaIKgyjuWG71ybDiYIwNJVJ2UmxRI&limit=100as", nil) + lazyLoadReq, _ := http.NewRequest("GET", "https://localhost:8800//_matrix/client/r0/rooms/!kvEtX3rFamfwKHO3:localhost:8800/context/%24GjmkRbajRHy8_cxcSbUU4qF_njV8yHeLphI2azTrPaI?limit=2&filter=%7B+%22lazy_load_members%22+%3A+true+%7D&access_token=t1Njzm74w3G40CJ5xrlf1V2haXom0z0Iq1qyyVWhbVo", nil) + invalidFilterReq, _ := http.NewRequest("GET", "https://localhost:8800//_matrix/client/r0/rooms/!kvEtX3rFamfwKHO3:localhost:8800/context/%24GjmkRbajRHy8_cxcSbUU4qF_njV8yHeLphI2azTrPaI?limit=2&filter=%7B+%22lazy_load_members%22+%3A+true&access_token=t1Njzm74w3G40CJ5xrlf1V2haXom0z0Iq1qyyVWhbVo", nil) + tests := []struct { + name string + req *http.Request + wantFilter *gomatrixserverlib.RoomEventFilter + wantErr bool + }{ + { + name: "no params set", + req: noParamsReq, + wantFilter: &gomatrixserverlib.RoomEventFilter{Limit: 10}, + }, + { + name: "limit 2 param set", + req: limit2Req, + wantFilter: &gomatrixserverlib.RoomEventFilter{Limit: 2}, + }, + { + name: "limit 10000 param set", + req: limit10000Req, + wantFilter: &gomatrixserverlib.RoomEventFilter{Limit: 100}, + }, + { + name: "filter lazy_load_members param set", + req: lazyLoadReq, + wantFilter: &gomatrixserverlib.RoomEventFilter{Limit: 2, LazyLoadMembers: true}, + }, + { + name: "invalid limit req", + req: invalidLimitReq, + wantErr: true, + }, + { + name: "invalid filter req", + req: invalidFilterReq, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotFilter, err := parseContextParams(tt.req) + if (err != nil) != tt.wantErr { + t.Errorf("parseContextParams() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotFilter, tt.wantFilter) { + t.Errorf("parseContextParams() gotFilter = %v, want %v", gotFilter, tt.wantFilter) + } + }) + } +} diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index 005a33555..be366ba10 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -77,4 +77,19 @@ func Setup( v3mux.Handle("/keys/changes", httputil.MakeAuthAPI("keys_changes", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return srp.OnIncomingKeyChangeRequest(req, device) })).Methods(http.MethodGet, http.MethodOptions) + + v3mux.Handle("/rooms/{roomId}/context/{eventId}", + httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + + return Context( + req, device, + rsAPI, syncDB, + vars["roomId"], vars["eventId"], + ) + }), + ).Methods(http.MethodGet, http.MethodOptions) } diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 4dd2bbc45..e44766338 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -145,4 +145,8 @@ type Database interface { // GetUserUnreadNotificationCounts returns statistics per room a user is interested in. GetUserUnreadNotificationCounts(ctx context.Context, userID string, from, to types.StreamPosition) (map[string]*eventutil.NotificationData, error) + + SelectContextEvent(ctx context.Context, roomID, eventID string) (int, gomatrixserverlib.HeaderedEvent, error) + SelectContextBeforeEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) ([]*gomatrixserverlib.HeaderedEvent, error) + SelectContextAfterEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error) } diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index 44de02c92..d4cc4f3fb 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -130,6 +130,25 @@ const selectStateInRangeSQL = "" + const deleteEventsForRoomSQL = "" + "DELETE FROM syncapi_output_room_events WHERE room_id = $1" +const selectContextEventSQL = "" + + "SELECT id, headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND event_id = $2" + +const selectContextBeforeEventSQL = "" + + "SELECT headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND id < $2" + + " AND ( $4::text[] IS NULL OR sender = ANY($4) )" + + " AND ( $5::text[] IS NULL OR NOT(sender = ANY($5)) )" + + " AND ( $6::text[] IS NULL OR type LIKE ANY($6) )" + + " AND ( $7::text[] IS NULL OR NOT(type LIKE ANY($7)) )" + + " ORDER BY id DESC LIMIT $3" + +const selectContextAfterEventSQL = "" + + "SELECT id, headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND id > $2" + + " AND ( $4::text[] IS NULL OR sender = ANY($4) )" + + " AND ( $5::text[] IS NULL OR NOT(sender = ANY($5)) )" + + " AND ( $6::text[] IS NULL OR type LIKE ANY($6) )" + + " AND ( $7::text[] IS NULL OR NOT(type LIKE ANY($7)) )" + + " ORDER BY id ASC LIMIT $3" + type outputRoomEventsStatements struct { insertEventStmt *sql.Stmt selectEventsStmt *sql.Stmt @@ -140,6 +159,9 @@ type outputRoomEventsStatements struct { selectStateInRangeStmt *sql.Stmt updateEventJSONStmt *sql.Stmt deleteEventsForRoomStmt *sql.Stmt + selectContextEventStmt *sql.Stmt + selectContextBeforeEventStmt *sql.Stmt + selectContextAfterEventStmt *sql.Stmt } func NewPostgresEventsTable(db *sql.DB) (tables.Events, error) { @@ -148,34 +170,20 @@ func NewPostgresEventsTable(db *sql.DB) (tables.Events, error) { if err != nil { return nil, err } - if s.insertEventStmt, err = db.Prepare(insertEventSQL); err != nil { - return nil, err - } - if s.selectEventsStmt, err = db.Prepare(selectEventsSQL); err != nil { - return nil, err - } - if s.selectMaxEventIDStmt, err = db.Prepare(selectMaxEventIDSQL); err != nil { - return nil, err - } - if s.selectRecentEventsStmt, err = db.Prepare(selectRecentEventsSQL); err != nil { - return nil, err - } - if s.selectRecentEventsForSyncStmt, err = db.Prepare(selectRecentEventsForSyncSQL); err != nil { - return nil, err - } - if s.selectEarlyEventsStmt, err = db.Prepare(selectEarlyEventsSQL); err != nil { - return nil, err - } - if s.selectStateInRangeStmt, err = db.Prepare(selectStateInRangeSQL); err != nil { - return nil, err - } - if s.updateEventJSONStmt, err = db.Prepare(updateEventJSONSQL); err != nil { - return nil, err - } - if s.deleteEventsForRoomStmt, err = db.Prepare(deleteEventsForRoomSQL); err != nil { - return nil, err - } - return s, nil + return s, sqlutil.StatementList{ + {&s.insertEventStmt, insertEventSQL}, + {&s.selectEventsStmt, selectEventsSQL}, + {&s.selectMaxEventIDStmt, selectMaxEventIDSQL}, + {&s.selectRecentEventsStmt, selectRecentEventsSQL}, + {&s.selectRecentEventsForSyncStmt, selectRecentEventsForSyncSQL}, + {&s.selectEarlyEventsStmt, selectEarlyEventsSQL}, + {&s.selectStateInRangeStmt, selectStateInRangeSQL}, + {&s.updateEventJSONStmt, updateEventJSONSQL}, + {&s.deleteEventsForRoomStmt, deleteEventsForRoomSQL}, + {&s.selectContextEventStmt, selectContextEventSQL}, + {&s.selectContextBeforeEventStmt, selectContextBeforeEventSQL}, + {&s.selectContextAfterEventStmt, selectContextAfterEventSQL}, + }.Prepare(db) } func (s *outputRoomEventsStatements) UpdateEventJSON(ctx context.Context, event *gomatrixserverlib.HeaderedEvent) error { @@ -436,6 +444,84 @@ func (s *outputRoomEventsStatements) DeleteEventsForRoom( return err } +func (s *outputRoomEventsStatements) SelectContextEvent(ctx context.Context, txn *sql.Tx, roomID, eventID string) (id int, evt gomatrixserverlib.HeaderedEvent, err error) { + row := sqlutil.TxStmt(txn, s.selectContextEventStmt).QueryRowContext(ctx, roomID, eventID) + + var eventAsString string + if err = row.Scan(&id, &eventAsString); err != nil { + return 0, evt, err + } + + if err = json.Unmarshal([]byte(eventAsString), &evt); err != nil { + return 0, evt, err + } + return id, evt, nil +} + +func (s *outputRoomEventsStatements) SelectContextBeforeEvent( + ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter, +) (evts []*gomatrixserverlib.HeaderedEvent, err error) { + rows, err := sqlutil.TxStmt(txn, s.selectContextBeforeEventStmt).QueryContext( + ctx, roomID, id, filter.Limit, + pq.StringArray(filter.Senders), + pq.StringArray(filter.NotSenders), + pq.StringArray(filterConvertTypeWildcardToSQL(filter.Types)), + pq.StringArray(filterConvertTypeWildcardToSQL(filter.NotTypes)), + ) + if err != nil { + return + } + defer rows.Close() + + for rows.Next() { + var ( + eventBytes []byte + evt *gomatrixserverlib.HeaderedEvent + ) + if err = rows.Scan(&eventBytes); err != nil { + return evts, err + } + if err = json.Unmarshal(eventBytes, &evt); err != nil { + return evts, err + } + evts = append(evts, evt) + } + + return evts, rows.Err() +} + +func (s *outputRoomEventsStatements) SelectContextAfterEvent( + ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter, +) (lastID int, evts []*gomatrixserverlib.HeaderedEvent, err error) { + rows, err := sqlutil.TxStmt(txn, s.selectContextAfterEventStmt).QueryContext( + ctx, roomID, id, filter.Limit, + pq.StringArray(filter.Senders), + pq.StringArray(filter.NotSenders), + pq.StringArray(filterConvertTypeWildcardToSQL(filter.Types)), + pq.StringArray(filterConvertTypeWildcardToSQL(filter.NotTypes)), + ) + if err != nil { + return + } + defer rows.Close() + + for rows.Next() { + var ( + eventBytes []byte + evt *gomatrixserverlib.HeaderedEvent + ) + if err = rows.Scan(&lastID, &eventBytes); err != nil { + return 0, evts, err + } + if err = json.Unmarshal(eventBytes, &evt); err != nil { + return 0, evts, err + } + evts = append(evts, evt) + } + + return lastID, evts, rows.Err() +} + func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { var result []types.StreamEvent for rows.Next() { diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index ca87969e4..87d7c6df7 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -976,3 +976,14 @@ func (d *Database) UpsertRoomUnreadNotificationCounts(ctx context.Context, userI func (d *Database) GetUserUnreadNotificationCounts(ctx context.Context, userID string, from, to types.StreamPosition) (map[string]*eventutil.NotificationData, error) { return d.NotificationData.SelectUserUnreadCounts(ctx, userID, from, to) } + +func (s *Database) SelectContextEvent(ctx context.Context, roomID, eventID string) (int, gomatrixserverlib.HeaderedEvent, error) { + return s.OutputEvents.SelectContextEvent(ctx, nil, roomID, eventID) +} + +func (s *Database) SelectContextBeforeEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) ([]*gomatrixserverlib.HeaderedEvent, error) { + return s.OutputEvents.SelectContextBeforeEvent(ctx, nil, id, roomID, filter) +} +func (s *Database) SelectContextAfterEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error) { + return s.OutputEvents.SelectContextAfterEvent(ctx, nil, id, roomID, filter) +} diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index 4fbbf45cf..c91ca6923 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -68,7 +68,8 @@ const selectRoomIDsWithMembershipSQL = "" + const selectCurrentStateSQL = "" + "SELECT event_id, headered_event_json FROM syncapi_current_room_state WHERE room_id = $1" - // WHEN, ORDER BY and LIMIT will be added by prepareWithFilter + +// WHEN, ORDER BY and LIMIT will be added by prepareWithFilter const selectJoinedUsersSQL = "" + "SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join'" diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index afdbe55ce..581ee6928 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -62,17 +62,17 @@ const selectEventsSQL = "" + const selectRecentEventsSQL = "" + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3" - // WHEN, ORDER BY and LIMIT are appended by prepareWithFilters +// WHEN, ORDER BY and LIMIT are appended by prepareWithFilters const selectRecentEventsForSyncSQL = "" + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3 AND exclude_from_sync = FALSE" - // WHEN, ORDER BY and LIMIT are appended by prepareWithFilters +// WHEN, ORDER BY and LIMIT are appended by prepareWithFilters const selectEarlyEventsSQL = "" + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3" - // WHEN, ORDER BY and LIMIT are appended by prepareWithFilters +// WHEN, ORDER BY and LIMIT are appended by prepareWithFilters const selectMaxEventIDSQL = "" + "SELECT MAX(id) FROM syncapi_output_room_events" @@ -85,19 +85,33 @@ const selectStateInRangeSQL = "" + " FROM syncapi_output_room_events" + " WHERE (id > $1 AND id <= $2)" + " AND ((add_state_ids IS NOT NULL AND add_state_ids != '') OR (remove_state_ids IS NOT NULL AND remove_state_ids != ''))" - // WHEN, ORDER BY and LIMIT are appended by prepareWithFilters +// WHEN, ORDER BY and LIMIT are appended by prepareWithFilters const deleteEventsForRoomSQL = "" + "DELETE FROM syncapi_output_room_events WHERE room_id = $1" +const selectContextEventSQL = "" + + "SELECT id, headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND event_id = $2" + +const selectContextBeforeEventSQL = "" + + "SELECT headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND id < $2" +// WHEN, ORDER BY and LIMIT are appended by prepareWithFilters + +const selectContextAfterEventSQL = "" + + "SELECT id, headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND id > $2" +// WHEN, ORDER BY and LIMIT are appended by prepareWithFilters + type outputRoomEventsStatements struct { - db *sql.DB - streamIDStatements *streamIDStatements - insertEventStmt *sql.Stmt - selectEventsStmt *sql.Stmt - selectMaxEventIDStmt *sql.Stmt - updateEventJSONStmt *sql.Stmt - deleteEventsForRoomStmt *sql.Stmt + db *sql.DB + streamIDStatements *streamIDStatements + insertEventStmt *sql.Stmt + selectEventsStmt *sql.Stmt + selectMaxEventIDStmt *sql.Stmt + updateEventJSONStmt *sql.Stmt + deleteEventsForRoomStmt *sql.Stmt + selectContextEventStmt *sql.Stmt + selectContextBeforeEventStmt *sql.Stmt + selectContextAfterEventStmt *sql.Stmt } func NewSqliteEventsTable(db *sql.DB, streamID *streamIDStatements) (tables.Events, error) { @@ -109,22 +123,16 @@ func NewSqliteEventsTable(db *sql.DB, streamID *streamIDStatements) (tables.Even if err != nil { return nil, err } - if s.insertEventStmt, err = db.Prepare(insertEventSQL); err != nil { - return nil, err - } - if s.selectEventsStmt, err = db.Prepare(selectEventsSQL); err != nil { - return nil, err - } - if s.selectMaxEventIDStmt, err = db.Prepare(selectMaxEventIDSQL); err != nil { - return nil, err - } - if s.updateEventJSONStmt, err = db.Prepare(updateEventJSONSQL); err != nil { - return nil, err - } - if s.deleteEventsForRoomStmt, err = db.Prepare(deleteEventsForRoomSQL); err != nil { - return nil, err - } - return s, nil + return s, sqlutil.StatementList{ + {&s.insertEventStmt, insertEventSQL}, + {&s.selectEventsStmt, selectEventsSQL}, + {&s.selectMaxEventIDStmt, selectMaxEventIDSQL}, + {&s.updateEventJSONStmt, updateEventJSONSQL}, + {&s.deleteEventsForRoomStmt, deleteEventsForRoomSQL}, + {&s.selectContextEventStmt, selectContextEventSQL}, + {&s.selectContextBeforeEventStmt, selectContextBeforeEventSQL}, + {&s.selectContextAfterEventStmt, selectContextAfterEventSQL}, + }.Prepare(db) } func (s *outputRoomEventsStatements) UpdateEventJSON(ctx context.Context, event *gomatrixserverlib.HeaderedEvent) error { @@ -462,6 +470,91 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { } return result, nil } +func (s *outputRoomEventsStatements) SelectContextEvent( + ctx context.Context, txn *sql.Tx, roomID, eventID string, +) (id int, evt gomatrixserverlib.HeaderedEvent, err error) { + row := sqlutil.TxStmt(txn, s.selectContextEventStmt).QueryRowContext(ctx, roomID, eventID) + var eventAsString string + if err = row.Scan(&id, &eventAsString); err != nil { + return 0, evt, err + } + + if err = json.Unmarshal([]byte(eventAsString), &evt); err != nil { + return 0, evt, err + } + return id, evt, nil +} + +func (s *outputRoomEventsStatements) SelectContextBeforeEvent( + ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter, +) (evts []*gomatrixserverlib.HeaderedEvent, err error) { + stmt, params, err := prepareWithFilters( + s.db, txn, selectContextBeforeEventSQL, + []interface{}{ + roomID, id, + }, + filter.Senders, filter.NotSenders, + filter.Types, filter.NotTypes, + nil, filter.Limit, FilterOrderDesc, + ) + + rows, err := stmt.QueryContext(ctx, params...) + if err != nil { + return + } + defer rows.Close() + + for rows.Next() { + var ( + eventBytes []byte + evt *gomatrixserverlib.HeaderedEvent + ) + if err = rows.Scan(&eventBytes); err != nil { + return evts, err + } + if err = json.Unmarshal(eventBytes, &evt); err != nil { + return evts, err + } + evts = append(evts, evt) + } + + return evts, rows.Err() +} + +func (s *outputRoomEventsStatements) SelectContextAfterEvent( + ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter, +) (lastID int, evts []*gomatrixserverlib.HeaderedEvent, err error) { + stmt, params, err := prepareWithFilters( + s.db, txn, selectContextAfterEventSQL, + []interface{}{ + roomID, id, + }, + filter.Senders, filter.NotSenders, + filter.Types, filter.NotTypes, + nil, filter.Limit, FilterOrderAsc, + ) + + rows, err := stmt.QueryContext(ctx, params...) + if err != nil { + return + } + defer rows.Close() + + for rows.Next() { + var ( + eventBytes []byte + evt *gomatrixserverlib.HeaderedEvent + ) + if err = rows.Scan(&lastID, &eventBytes); err != nil { + return 0, evts, err + } + if err = json.Unmarshal(eventBytes, &evt); err != nil { + return 0, evts, err + } + evts = append(evts, evt) + } + return lastID, evts, rows.Err() +} func unmarshalStateIDs(addIDsJSON, delIDsJSON string) (addIDs []string, delIDs []string, err error) { if len(addIDsJSON) > 0 { diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index f2fa7a254..1ebb42651 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -64,6 +64,10 @@ type Events interface { UpdateEventJSON(ctx context.Context, event *gomatrixserverlib.HeaderedEvent) error // DeleteEventsForRoom removes all event information for a room. This should only be done when removing the room entirely. DeleteEventsForRoom(ctx context.Context, txn *sql.Tx, roomID string) (err error) + + SelectContextEvent(ctx context.Context, txn *sql.Tx, roomID, eventID string) (int, gomatrixserverlib.HeaderedEvent, error) + SelectContextBeforeEvent(ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) ([]*gomatrixserverlib.HeaderedEvent, error) + SelectContextAfterEvent(ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error) } // Topology keeps track of the depths and stream positions for all events. diff --git a/sytest-blacklist b/sytest-blacklist index 3e08f0cb4..16abce8da 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -24,11 +24,8 @@ Local device key changes get to remote servers with correct prev_id # Flakey Local device key changes appear in /keys/changes -Device list doesn't change if remote server is down # we don't support groups Remove group category Remove group role -# See https://github.com/matrix-org/sytest/pull/1142 -Device list doesn't change if remote server is down diff --git a/sytest-whitelist b/sytest-whitelist index c1d0ee3bd..a1d24ec27 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -648,3 +648,11 @@ If a device list update goes missing, the server resyncs on the next one uploading self-signing key notifies over federation uploading signed devices gets propagated over federation Device list doesn't change if remote server is down +/context/ on joined room works +/context/ on non world readable room does not work +/context/ returns correct number of events +/context/ with lazy_load_members filter works +Can query remote device keys using POST after notification +Device deletion propagates over federation +Get left notifs in sync and /keys/changes when other user leaves +Remote banned user is kicked and may not rejoin until unbanned