diff --git a/.gitignore b/.gitignore index 2a8c2cf55..e4f0112c4 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,10 @@ _testmain.go *.test *.prof *.wasm +*.aar +*.jar +*.framework +*.xcframework # Generated keys *.pem @@ -65,4 +69,7 @@ test/wasm/node_modules # Ignore complement folder when running locally complement/ +# Stuff from GitHub Pages +docs/_site + media_store/ diff --git a/CHANGES.md b/CHANGES.md index 6278bcba4..3deebd8a0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,64 @@ # Changelog +## Dendrite 0.8.5 (2022-05-13) + +### Features + +* New living documentation available at , including new installation instructions +* The built-in NATS Server has been updated to version 2.8.2 + +### Fixes + +* Monolith deployments will no longer panic at startup if given a config file that does not include the `internal_api` and `external_api` options +* State resolution v2 now correctly identifies other events related to power events, which should fix some event auth issues +* The latest events updater will no longer implicitly trust the new forward extremities when calculating the current room state, which may help to avoid some state resets +* The one-time key count is now correctly returned in `/sync` even if the request otherwise timed out, which should reduce the chance that unnecessary one-time keys will be uploaded by clients +* The `create-account` tool should now work properly when the database is configured using the global connection pool + +## Dendrite 0.8.4 (2022-05-10) + +### Fixes + +* Fixes a regression introduced in the previous version where appservices, push and phone-home statistics would not work over plain HTTP +* Adds missing indexes to the sync API output events table, which should significantly improve `/sync` performance and reduce database CPU usage +* Building Dendrite with the `bimg` thumbnailer should now work again (contributed by [database64128](https://github.com/database64128)) + +## Dendrite 0.8.3 (2022-05-09) + +### Features + +* Open registration is now harder to enable, which should reduce the chance that Dendrite servers will be used to conduct spam or abuse attacks + * Dendrite will only enable open registration if you pass the `--really-enable-open-registration` command line flag at startup + * If open registration is enabled but this command line flag is not passed, Dendrite will fail to start up +* Dendrite now supports phone-home statistic reporting + * These statistics include things like the number of registered and active users, some configuration options and platform/environment details, to help us to understand how Dendrite is used + * This is not enabled by default — it must be enabled in the `global.report_stats` section of the config file +* Monolith installations can now be configured with a single global database connection pool (in `global.database` in the config) rather than having to configure each component separately + * This also means that you no longer need to balance connection counts between different components, as they will share the same larger pool + * Specific components can override the global database settings by specifying their own `database` block + * To use only the global pool, you must configure `global.database` and then remove the `database` block from all of the component sections of the config file +* A new admin API endpoint `/_dendrite/admin/evacuateRoom/{roomID}` has been added, allowing server admins to forcefully part all local users from a given room +* The sync notifier now only loads members for the relevant rooms, which should reduce CPU usage and load on the database +* A number of component interfaces have been refactored for cleanliness and developer ease +* Event auth errors in the log should now be much more useful, including the reason for the event failures +* The forward extremity calculation in the roomserver has been simplified +* A new index has been added to the one-time keys table in the keyserver which should speed up key count lookups + +### Fixes + +* Dendrite will no longer process events for rooms where there are no local users joined, which should help to reduce CPU and RAM usage +* A bug has been fixed in event auth when changing the user levels in `m.room.power_levels` events +* Usernames should no longer be duplicated when no room name is set +* Device display names should now be correctly propagated over federation +* A panic when uploading cross-signing signatures has been fixed +* Presence is now correctly limited in `/sync` based on the filters +* The presence stream position returned by `/sync` will now be correct if no presence events were returned +* The media `/config` endpoint will no longer return a maximum upload size field if it is configured to be unlimited in the Dendrite config +* The server notices room will no longer produce "User is already joined to the room" errors +* Consumer errors will no longer flood the logs during a graceful shutdown +* Sync API and federation API consumers will no longer unnecessarily query added state events matching the one in the output event +* The Sync API will no longer unnecessarily track invites for remote users + ## Dendrite 0.8.2 (2022-04-27) ### Features diff --git a/README.md b/README.md index 2a8c36508..e8b7bd0e2 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # Dendrite + [![Build status](https://github.com/matrix-org/dendrite/actions/workflows/dendrite.yml/badge.svg?event=push)](https://github.com/matrix-org/dendrite/actions/workflows/dendrite.yml) [![Dendrite](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) [![Dendrite Dev](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) Dendrite is a second-generation Matrix homeserver written in Go. @@ -52,7 +53,7 @@ The [Federation Tester](https://federationtester.matrix.org) can be used to veri ## Get started -If you wish to build a fully-federating Dendrite instance, see [INSTALL.md](docs/INSTALL.md). For running in Docker, see [build/docker](build/docker). +If you wish to build a fully-federating Dendrite instance, see [the Installation documentation](https://matrix-org.github.io/dendrite/installation). For running in Docker, see [build/docker](build/docker). The following instructions are enough to get Dendrite started as a non-federating test deployment using self-signed certificates and SQLite databases: @@ -70,7 +71,7 @@ $ ./bin/generate-keys --tls-cert server.crt --tls-key server.key # Copy and modify the config file - you'll need to set a server name and paths to the keys # at the very least, along with setting up the database connection strings. -$ cp dendrite-config.yaml dendrite.yaml +$ cp dendrite-sample.monolith.yaml dendrite.yaml # Build and run the server: $ ./bin/dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml diff --git a/appservice/appservice.go b/appservice/appservice.go index c5ae9ceb2..8fe1b2fc4 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -16,6 +16,8 @@ package appservice import ( "context" + "crypto/tls" + "net/http" "sync" "time" @@ -33,7 +35,6 @@ import ( "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" ) // AddInternalRoutes registers HTTP handlers for internal API calls @@ -45,15 +46,19 @@ func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceInte // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( base *base.BaseDendrite, - userAPI userapi.AppserviceUserAPI, - rsAPI roomserverAPI.AppserviceRoomserverAPI, + userAPI userapi.UserInternalAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, ) appserviceAPI.AppServiceInternalAPI { - client := gomatrixserverlib.NewClient( - gomatrixserverlib.WithTimeout(time.Second*30), - gomatrixserverlib.WithKeepAlives(false), - gomatrixserverlib.WithSkipVerify(base.Cfg.AppServiceAPI.DisableTLSValidation), - ) - + client := &http.Client{ + Timeout: time.Second * 30, + Transport: &http.Transport{ + DisableKeepAlives: true, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: base.Cfg.AppServiceAPI.DisableTLSValidation, + }, + Proxy: http.ProxyFromEnvironment, + }, + } js, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) // Create a connection to the appservice postgres DB diff --git a/appservice/query/query.go b/appservice/query/query.go index b7b0b335a..dacd3caa8 100644 --- a/appservice/query/query.go +++ b/appservice/query/query.go @@ -23,7 +23,6 @@ import ( "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/gomatrixserverlib" opentracing "github.com/opentracing/opentracing-go" log "github.com/sirupsen/logrus" ) @@ -33,7 +32,7 @@ const userIDExistsPath = "/users/" // AppServiceQueryAPI is an implementation of api.AppServiceQueryAPI type AppServiceQueryAPI struct { - HTTPClient *gomatrixserverlib.Client + HTTPClient *http.Client Cfg *config.Dendrite } @@ -65,8 +64,9 @@ func (a *AppServiceQueryAPI) RoomAliasExists( if err != nil { return err } + req = req.WithContext(ctx) - resp, err := a.HTTPClient.DoHTTPRequest(ctx, req) + resp, err := a.HTTPClient.Do(req) if resp != nil { defer func() { err = resp.Body.Close() @@ -130,7 +130,7 @@ func (a *AppServiceQueryAPI) UserIDExists( if err != nil { return err } - resp, err := a.HTTPClient.DoHTTPRequest(ctx, req) + resp, err := a.HTTPClient.Do(req.WithContext(ctx)) if resp != nil { defer func() { err = resp.Body.Close() diff --git a/appservice/workers/transaction_scheduler.go b/appservice/workers/transaction_scheduler.go index 47d447c2c..4dab00bd7 100644 --- a/appservice/workers/transaction_scheduler.go +++ b/appservice/workers/transaction_scheduler.go @@ -42,7 +42,7 @@ var ( // size), then send that off to the AS's /transactions/{txnID} endpoint. It also // handles exponentially backing off in case the AS isn't currently available. func SetupTransactionWorkers( - client *gomatrixserverlib.Client, + client *http.Client, appserviceDB storage.Database, workerStates []types.ApplicationServiceWorkerState, ) error { @@ -58,7 +58,7 @@ func SetupTransactionWorkers( // worker is a goroutine that sends any queued events to the application service // it is given. -func worker(client *gomatrixserverlib.Client, db storage.Database, ws types.ApplicationServiceWorkerState) { +func worker(client *http.Client, db storage.Database, ws types.ApplicationServiceWorkerState) { log.WithFields(log.Fields{ "appservice": ws.AppService.ID, }).Info("Starting application service") @@ -200,7 +200,7 @@ func createTransaction( // send sends events to an application service. Returns an error if an OK was not // received back from the application service or the request timed out. func send( - client *gomatrixserverlib.Client, + client *http.Client, appservice config.ApplicationService, txnID int, transaction []byte, @@ -213,7 +213,7 @@ func send( return err } req.Header.Set("Content-Type", "application/json") - resp, err := client.DoHTTPRequest(context.TODO(), req) + resp, err := client.Do(req) if err != nil { return err } diff --git a/build/docker/README.md b/build/docker/README.md index 7425d96cb..261519fde 100644 --- a/build/docker/README.md +++ b/build/docker/README.md @@ -27,8 +27,7 @@ There are three sample `docker-compose` files: The `docker-compose` files refer to the `/etc/dendrite` volume as where the runtime config should come from. The mounted folder must contain: -- `dendrite.yaml` configuration file (from the [Docker config folder](https://github.com/matrix-org/dendrite/tree/master/build/docker/config) - sample in the `build/docker/config` folder of this repository.) +- `dendrite.yaml` configuration file (based on one of the sample config files) - `matrix_key.pem` server key, as generated using `cmd/generate-keys` - `server.crt` certificate file - `server.key` private key file for the above certificate @@ -49,7 +48,7 @@ The key files will now exist in your current working directory, and can be mount ## Starting Dendrite as a monolith deployment -Create your config based on the [`dendrite.yaml`](https://github.com/matrix-org/dendrite/tree/master/build/docker/config) configuration file in the `build/docker/config` folder of this repository. +Create your config based on the [`dendrite-sample.monolith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.monolith.yaml) sample configuration file. Then start the deployment: @@ -59,7 +58,7 @@ docker-compose -f docker-compose.monolith.yml up ## Starting Dendrite as a polylith deployment -Create your config based on the [`dendrite-config.yaml`](https://github.com/matrix-org/dendrite/tree/master/build/docker/config) configuration file in the `build/docker/config` folder of this repository. +Create your config based on the [`dendrite-sample.polylith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml) sample configuration file. Then start the deployment: diff --git a/build/docker/config/dendrite.yaml b/build/docker/config/dendrite.yaml deleted file mode 100644 index 94dcf4558..000000000 --- a/build/docker/config/dendrite.yaml +++ /dev/null @@ -1,348 +0,0 @@ -# This is the Dendrite configuration file. -# -# The configuration is split up into sections - each Dendrite component has a -# configuration section, in addition to the "global" section which applies to -# all components. -# -# At a minimum, to get started, you will need to update the settings in the -# "global" section for your deployment, and you will need to check that the -# database "connection_string" line in each component section is correct. -# -# Each component with a "database" section can accept the following formats -# for "connection_string": -# SQLite: file:filename.db -# file:///path/to/filename.db -# PostgreSQL: postgresql://user:pass@hostname/database?params=... -# -# SQLite is embedded into Dendrite and therefore no further prerequisites are -# needed for the database when using SQLite mode. However, performance with -# PostgreSQL is significantly better and recommended for multi-user deployments. -# SQLite is typically around 20-30% slower than PostgreSQL when tested with a -# small number of users and likely will perform worse still with a higher volume -# of users. -# -# The "max_open_conns" and "max_idle_conns" settings configure the maximum -# number of open/idle database connections. The value 0 will use the database -# engine default, and a negative value will use unlimited connections. The -# "conn_max_lifetime" option controls the maximum length of time a database -# connection can be idle in seconds - a negative value is unlimited. - -# The version of the configuration file. -version: 2 - -# Global Matrix configuration. This configuration applies to all components. -global: - # The domain name of this homeserver. - server_name: example.com - - # The path to the signing private key file, used to sign requests and events. - private_key: matrix_key.pem - - # The paths and expiry timestamps (as a UNIX timestamp in millisecond precision) - # to old signing private keys that were formerly in use on this domain. These - # keys will not be used for federation request or event signing, but will be - # provided to any other homeserver that asks when trying to verify old events. - # old_private_keys: - # - private_key: old_matrix_key.pem - # expired_at: 1601024554498 - - # How long a remote server can cache our server signing key before requesting it - # again. Increasing this number will reduce the number of requests made by other - # servers for our key but increases the period that a compromised key will be - # considered valid by other homeservers. - key_validity_period: 168h0m0s - - # The server name to delegate server-server communications to, with optional port - # e.g. localhost:443 - well_known_server_name: "" - - # Lists of domains that the server will trust as identity servers to verify third - # party identifiers such as phone numbers and email addresses. - trusted_third_party_id_servers: - - matrix.org - - vector.im - - # Disables federation. Dendrite will not be able to make any outbound HTTP requests - # to other servers and the federation API will not be exposed. - disable_federation: false - - # Configures the handling of presence events. - presence: - # Whether inbound presence events are allowed, e.g. receiving presence events from other servers - enable_inbound: false - # Whether outbound presence events are allowed, e.g. sending presence events to other servers - enable_outbound: false - - # Configuration for NATS JetStream - jetstream: - # A list of NATS Server addresses to connect to. If none are specified, an - # internal NATS server will be started automatically when running Dendrite - # in monolith mode. It is required to specify the address of at least one - # NATS Server node if running in polylith mode. - addresses: - - jetstream:4222 - - # Keep all NATS streams in memory, rather than persisting it to the storage - # path below. This option is present primarily for integration testing and - # should not be used on a real world Dendrite deployment. - in_memory: false - - # Persistent directory to store JetStream streams in. This directory - # should be preserved across Dendrite restarts. - storage_path: ./ - - # The prefix to use for stream names for this homeserver - really only - # useful if running more than one Dendrite on the same NATS deployment. - topic_prefix: Dendrite - - # Configuration for Prometheus metric collection. - metrics: - # Whether or not Prometheus metrics are enabled. - enabled: false - - # HTTP basic authentication to protect access to monitoring. - basic_auth: - username: metrics - password: metrics - - # DNS cache options. The DNS cache may reduce the load on DNS servers - # if there is no local caching resolver available for use. - dns_cache: - # Whether or not the DNS cache is enabled. - enabled: false - - # Maximum number of entries to hold in the DNS cache, and - # for how long those items should be considered valid in seconds. - cache_size: 256 - cache_lifetime: 300 - -# Configuration for the Appservice API. -app_service_api: - internal_api: - listen: http://0.0.0.0:7777 - connect: http://appservice_api:7777 - database: - connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_appservice?sslmode=disable - max_open_conns: 10 - max_idle_conns: 2 - conn_max_lifetime: -1 - - # Appservice configuration files to load into this homeserver. - config_files: [] - -# Configuration for the Client API. -client_api: - internal_api: - listen: http://0.0.0.0:7771 - connect: http://client_api:7771 - external_api: - listen: http://0.0.0.0:8071 - - # Prevents new users from being able to register on this homeserver, except when - # using the registration shared secret below. - registration_disabled: true - - # If set, allows registration by anyone who knows the shared secret, regardless of - # whether registration is otherwise disabled. - registration_shared_secret: "" - - # Whether to require reCAPTCHA for registration. - enable_registration_captcha: false - - # Settings for ReCAPTCHA. - recaptcha_public_key: "" - recaptcha_private_key: "" - recaptcha_bypass_secret: "" - recaptcha_siteverify_api: "" - - # TURN server information that this homeserver should send to clients. - turn: - turn_user_lifetime: "" - turn_uris: [] - turn_shared_secret: "" - turn_username: "" - turn_password: "" - - # Settings for rate-limited endpoints. Rate limiting will kick in after the - # threshold number of "slots" have been taken by requests from a specific - # host. Each "slot" will be released after the cooloff time in milliseconds. - rate_limiting: - enabled: true - threshold: 5 - cooloff_ms: 500 - -# Configuration for the Federation API. -federation_api: - internal_api: - listen: http://0.0.0.0:7772 - connect: http://federation_api:7772 - external_api: - listen: http://0.0.0.0:8072 - database: - connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_federationapi?sslmode=disable - max_open_conns: 10 - max_idle_conns: 2 - conn_max_lifetime: -1 - - # How many times we will try to resend a failed transaction to a specific server. The - # backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds etc. - send_max_retries: 16 - - # Disable the validation of TLS certificates of remote federated homeservers. Do not - # enable this option in production as it presents a security risk! - disable_tls_validation: false - - # Use the following proxy server for outbound federation traffic. - proxy_outbound: - enabled: false - protocol: http - host: localhost - port: 8080 - - # Perspective keyservers to use as a backup when direct key fetches fail. This may - # be required to satisfy key requests for servers that are no longer online when - # joining some rooms. - key_perspectives: - - server_name: matrix.org - keys: - - key_id: ed25519:auto - public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw - - key_id: ed25519:a_RXGa - public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ - - # This option will control whether Dendrite will prefer to look up keys directly - # or whether it should try perspective servers first, using direct fetches as a - # last resort. - prefer_direct_fetch: false - -# Configuration for the Key Server (for end-to-end encryption). -key_server: - internal_api: - listen: http://0.0.0.0:7779 - connect: http://key_server:7779 - database: - connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_keyserver?sslmode=disable - max_open_conns: 10 - max_idle_conns: 2 - conn_max_lifetime: -1 - -# Configuration for the Media API. -media_api: - internal_api: - listen: http://0.0.0.0:7774 - connect: http://media_api:7774 - external_api: - listen: http://0.0.0.0:8074 - database: - connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_mediaapi?sslmode=disable - max_open_conns: 10 - max_idle_conns: 2 - conn_max_lifetime: -1 - - # Storage path for uploaded media. May be relative or absolute. - base_path: /var/dendrite/media - - # The maximum allowed file size (in bytes) for media uploads to this homeserver - # (0 = unlimited). - max_file_size_bytes: 10485760 - - # Whether to dynamically generate thumbnails if needed. - dynamic_thumbnails: false - - # The maximum number of simultaneous thumbnail generators to run. - max_thumbnail_generators: 10 - - # A list of thumbnail sizes to be generated for media content. - thumbnail_sizes: - - width: 32 - height: 32 - method: crop - - width: 96 - height: 96 - method: crop - - width: 640 - height: 480 - method: scale - -# Configuration for experimental MSC's -mscs: - # A list of enabled MSC's - # Currently valid values are: - # - msc2836 (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836) - # - msc2946 (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946) - mscs: [] - database: - connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_mscs?sslmode=disable - max_open_conns: 5 - max_idle_conns: 2 - conn_max_lifetime: -1 - -# Configuration for the Room Server. -room_server: - internal_api: - listen: http://0.0.0.0:7770 - connect: http://room_server:7770 - database: - connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_roomserver?sslmode=disable - max_open_conns: 10 - max_idle_conns: 2 - conn_max_lifetime: -1 - -# Configuration for the Sync API. -sync_api: - internal_api: - listen: http://0.0.0.0:7773 - connect: http://sync_api:7773 - external_api: - listen: http://0.0.0.0:8073 - database: - connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_syncapi?sslmode=disable - max_open_conns: 10 - max_idle_conns: 2 - conn_max_lifetime: -1 - -# Configuration for the User API. -user_api: - internal_api: - listen: http://0.0.0.0:7781 - connect: http://user_api:7781 - account_database: - connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_userapi_accounts?sslmode=disable - max_open_conns: 10 - max_idle_conns: 2 - conn_max_lifetime: -1 - -# Configuration for the Push Server API. -push_server: - internal_api: - listen: http://localhost:7782 - connect: http://localhost:7782 - database: - connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_pushserver?sslmode=disable - max_open_conns: 10 - max_idle_conns: 2 - conn_max_lifetime: -1 - -# Configuration for Opentracing. -# See https://github.com/matrix-org/dendrite/tree/master/docs/tracing for information on -# how this works and how to set it up. -tracing: - enabled: false - jaeger: - serviceName: "" - disabled: false - rpc_metrics: false - tags: [] - sampler: null - reporter: null - headers: null - baggage_restrictions: null - throttler: null - -# Logging configuration, in addition to the standard logging that is sent to -# stdout by Dendrite. -logging: - - type: file - level: info - params: - path: /var/log/dendrite diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 310ac7dda..664ca85d9 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -225,7 +225,7 @@ func (m *DendriteMonolith) Start() { pk = sk.Public().(ed25519.PublicKey) } - m.listener, err = net.Listen("tcp", ":65432") + m.listener, err = net.Listen("tcp", "localhost:65432") if err != nil { panic(err) } diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index cfdf6f2de..77f627eb2 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -188,6 +188,12 @@ func SendUnban( if err != nil { return util.ErrorResponse(err) } + if !queryRes.RoomExists { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("room does not exist"), + } + } // unban is only valid if the user is currently banned if queryRes.Membership != "ban" { return util.JSONResponse{ @@ -471,6 +477,12 @@ func SendForget( logger.WithError(err).Error("QueryMembershipForUser: could not query membership for user") return jsonerror.InternalServerError() } + if !membershipRes.RoomExists { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("room does not exist"), + } + } if membershipRes.IsInRoom { return util.JSONResponse{ Code: http.StatusBadRequest, diff --git a/clientapi/routing/state.go b/clientapi/routing/state.go index c6e9e91d0..12984c39a 100644 --- a/clientapi/routing/state.go +++ b/clientapi/routing/state.go @@ -56,6 +56,12 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a util.GetLogger(ctx).WithError(err).Error("queryAPI.QueryLatestEventsAndState failed") return jsonerror.InternalServerError() } + if !stateRes.RoomExists { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("room does not exist"), + } + } // Look at the room state and see if we have a history visibility event // that marks the room as world-readable. If we don't then we assume that diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index 6e7426a7f..9670fecad 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -231,7 +231,7 @@ func queryIDServerStoreInvite( profile = &authtypes.Profile{} } - client := gomatrixserverlib.NewClient() + client := http.Client{} data := url.Values{} data.Add("medium", body.Medium) @@ -253,7 +253,7 @@ func queryIDServerStoreInvite( } req.Header.Add("Content-Type", "application/x-www-form-urlencoded") - resp, err := client.DoHTTPRequest(ctx, req) + resp, err := client.Do(req.WithContext(ctx)) if err != nil { return nil, err } diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go index 50a5a9a5c..37103db16 100644 --- a/cmd/create-account/main.go +++ b/cmd/create-account/main.go @@ -25,6 +25,7 @@ import ( "strings" "github.com/matrix-org/dendrite/setup" + "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/storage" "github.com/sirupsen/logrus" @@ -99,8 +100,14 @@ func main() { } } + // avoid warning about open registration + cfg.ClientAPI.RegistrationDisabled = true + + b := base.NewBaseDendrite(cfg, "") + defer b.Close() // nolint: errcheck + accountDB, err := storage.NewUserAPIDatabase( - nil, + b, &cfg.UserAPI.AccountDatabase, cfg.Global.ServerName, cfg.UserAPI.BCryptCost, diff --git a/cmd/generate-keys/main.go b/cmd/generate-keys/main.go index bddf219dc..8acd28be0 100644 --- a/cmd/generate-keys/main.go +++ b/cmd/generate-keys/main.go @@ -20,7 +20,7 @@ import ( "log" "os" - "github.com/matrix-org/dendrite/internal/test" + "github.com/matrix-org/dendrite/test" ) const usage = `Usage: %s diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml new file mode 100644 index 000000000..e974dbcba --- /dev/null +++ b/dendrite-sample.monolith.yaml @@ -0,0 +1,279 @@ +# This is the Dendrite configuration file. +# +# The configuration is split up into sections - each Dendrite component has a +# configuration section, in addition to the "global" section which applies to +# all components. + +# The version of the configuration file. +version: 2 + +# Global Matrix configuration. This configuration applies to all components. +global: + # The domain name of this homeserver. + server_name: localhost + + # The path to the signing private key file, used to sign requests and events. + # Note that this is NOT the same private key as used for TLS! To generate a + # signing key, use "./bin/generate-keys --private-key matrix_key.pem". + private_key: matrix_key.pem + + # The paths and expiry timestamps (as a UNIX timestamp in millisecond precision) + # to old signing private keys that were formerly in use on this domain. These + # keys will not be used for federation request or event signing, but will be + # provided to any other homeserver that asks when trying to verify old events. + old_private_keys: + # - private_key: old_matrix_key.pem + # expired_at: 1601024554498 + + # How long a remote server can cache our server signing key before requesting it + # again. Increasing this number will reduce the number of requests made by other + # servers for our key but increases the period that a compromised key will be + # considered valid by other homeservers. + key_validity_period: 168h0m0s + + # Global database connection pool, for PostgreSQL monolith deployments only. If + # this section is populated then you can omit the "database" blocks in all other + # sections. For polylith deployments, or monolith deployments using SQLite databases, + # you must configure the "database" block for each component instead. + database: + connection_string: postgresql://username:password@hostname/dendrite?sslmode=disable + max_open_conns: 100 + max_idle_conns: 5 + conn_max_lifetime: -1 + + # The server name to delegate server-server communications to, with optional port + # e.g. localhost:443 + well_known_server_name: "" + + # Lists of domains that the server will trust as identity servers to verify third + # party identifiers such as phone numbers and email addresses. + trusted_third_party_id_servers: + - matrix.org + - vector.im + + # Disables federation. Dendrite will not be able to communicate with other servers + # in the Matrix federation and the federation API will not be exposed. + disable_federation: false + + # Configures the handling of presence events. Inbound controls whether we receive + # presence events from other servers, outbound controls whether we send presence + # events for our local users to other servers. + presence: + enable_inbound: false + enable_outbound: false + + # Configures phone-home statistics reporting. These statistics contain the server + # name, number of active users and some information on your deployment config. + # We use this information to understand how Dendrite is being used in the wild. + report_stats: + enabled: false + endpoint: https://matrix.org/report-usage-stats/push + + # Server notices allows server admins to send messages to all users on the server. + server_notices: + enabled: false + # The local part, display name and avatar URL (as a mxc:// URL) for the user that + # will send the server notices. These are visible to all users on the deployment. + local_part: "_server" + display_name: "Server Alerts" + avatar_url: "" + # The room name to be used when sending server notices. This room name will + # appear in user clients. + room_name: "Server Alerts" + + # Configuration for NATS JetStream + jetstream: + # A list of NATS Server addresses to connect to. If none are specified, an + # internal NATS server will be started automatically when running Dendrite in + # monolith mode. For polylith deployments, it is required to specify the address + # of at least one NATS Server node. + addresses: + # - localhost:4222 + + # Persistent directory to store JetStream streams in. This directory should be + # preserved across Dendrite restarts. + storage_path: ./ + + # The prefix to use for stream names for this homeserver - really only useful + # if you are running more than one Dendrite server on the same NATS deployment. + topic_prefix: Dendrite + + # Configuration for Prometheus metric collection. + metrics: + enabled: false + basic_auth: + username: metrics + password: metrics + + # Optional DNS cache. The DNS cache may reduce the load on DNS servers if there + # is no local caching resolver available for use. + dns_cache: + enabled: false + cache_size: 256 + cache_lifetime: "5m" # 5 minutes; https://pkg.go.dev/time@master#ParseDuration + +# Configuration for the Appservice API. +app_service_api: + # Disable the validation of TLS certificates of appservices. This is + # not recommended in production since it may allow appservice traffic + # to be sent to an insecure endpoint. + disable_tls_validation: false + + # Appservice configuration files to load into this homeserver. + config_files: + # - /path/to/appservice_registration.yaml + +# Configuration for the Client API. +client_api: + # Prevents new users from being able to register on this homeserver, except when + # using the registration shared secret below. + registration_disabled: true + + # Prevents new guest accounts from being created. Guest registration is also + # disabled implicitly by setting 'registration_disabled' above. + guests_disabled: true + + # If set, allows registration by anyone who knows the shared secret, regardless + # of whether registration is otherwise disabled. + registration_shared_secret: "" + + # Whether to require reCAPTCHA for registration. If you have enabled registration + # then this is HIGHLY RECOMMENDED to reduce the risk of your homeserver being used + # for coordinated spam attacks. + enable_registration_captcha: false + + # Settings for ReCAPTCHA. + recaptcha_public_key: "" + recaptcha_private_key: "" + recaptcha_bypass_secret: "" + recaptcha_siteverify_api: "" + + # TURN server information that this homeserver should send to clients. + turn: + turn_user_lifetime: "" + turn_uris: + # - turn:turn.server.org?transport=udp + # - turn:turn.server.org?transport=tcp + turn_shared_secret: "" + turn_username: "" + turn_password: "" + + # Settings for rate-limited endpoints. Rate limiting kicks in after the threshold + # number of "slots" have been taken by requests from a specific host. Each "slot" + # will be released after the cooloff time in milliseconds. + rate_limiting: + enabled: true + threshold: 5 + cooloff_ms: 500 + +# Configuration for the Federation API. +federation_api: + # How many times we will try to resend a failed transaction to a specific server. The + # backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds etc. Once + # the max retries are exceeded, Dendrite will no longer try to send transactions to + # that server until it comes back to life and connects to us again. + send_max_retries: 16 + + # Disable the validation of TLS certificates of remote federated homeservers. Do not + # enable this option in production as it presents a security risk! + disable_tls_validation: false + + # Perspective keyservers to use as a backup when direct key fetches fail. This may + # be required to satisfy key requests for servers that are no longer online when + # joining some rooms. + key_perspectives: + - server_name: matrix.org + keys: + - key_id: ed25519:auto + public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw + - key_id: ed25519:a_RXGa + public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ + + # This option will control whether Dendrite will prefer to look up keys directly + # or whether it should try perspective servers first, using direct fetches as a + # last resort. + prefer_direct_fetch: false + +# Configuration for the Media API. +media_api: + # Storage path for uploaded media. May be relative or absolute. + base_path: ./media_store + + # The maximum allowed file size (in bytes) for media uploads to this homeserver + # (0 = unlimited). If using a reverse proxy, ensure it allows requests at least + #this large (e.g. the client_max_body_size setting in nginx). + max_file_size_bytes: 10485760 + + # Whether to dynamically generate thumbnails if needed. + dynamic_thumbnails: false + + # The maximum number of simultaneous thumbnail generators to run. + max_thumbnail_generators: 10 + + # A list of thumbnail sizes to be generated for media content. + thumbnail_sizes: + - width: 32 + height: 32 + method: crop + - width: 96 + height: 96 + method: crop + - width: 640 + height: 480 + method: scale + +# Configuration for enabling experimental MSCs on this homeserver. +mscs: + mscs: + # - msc2836 # (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836) + # - msc2946 # (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946) + +# Configuration for the Sync API. +sync_api: + # This option controls which HTTP header to inspect to find the real remote IP + # address of the client. This is likely required if Dendrite is running behind + # a reverse proxy server. + # real_ip_header: X-Real-IP + +# Configuration for the User API. +user_api: + # The cost when hashing passwords on registration/login. Default: 10. Min: 4, Max: 31 + # See https://pkg.go.dev/golang.org/x/crypto/bcrypt for more information. + # Setting this lower makes registration/login consume less CPU resources at the cost + # of security should the database be compromised. Setting this higher makes registration/login + # consume more CPU resources but makes it harder to brute force password hashes. This value + # can be lowered if performing tests or on embedded Dendrite instances (e.g WASM builds). + bcrypt_cost: 10 + + # The length of time that a token issued for a relying party from + # /_matrix/client/r0/user/{userId}/openid/request_token endpoint + # is considered to be valid in milliseconds. + # The default lifetime is 3600000ms (60 minutes). + # openid_token_lifetime_ms: 3600000 + +# Configuration for Opentracing. +# See https://github.com/matrix-org/dendrite/tree/master/docs/tracing for information on +# how this works and how to set it up. +tracing: + enabled: false + jaeger: + serviceName: "" + disabled: false + rpc_metrics: false + tags: [] + sampler: null + reporter: null + headers: null + baggage_restrictions: null + throttler: null + +# Logging configuration. The "std" logging type controls the logs being sent to +# stdout. The "file" logging type controls logs being written to a log folder on +# the disk. Supported log levels are "debug", "info", "warn", "error". +logging: + - type: std + level: info + - type: file + level: info + params: + path: ./logs diff --git a/dendrite-config.yaml b/dendrite-sample.polylith.yaml similarity index 58% rename from dendrite-config.yaml rename to dendrite-sample.polylith.yaml index e9d14c287..dfbdab519 100644 --- a/dendrite-config.yaml +++ b/dendrite-sample.polylith.yaml @@ -3,29 +3,6 @@ # The configuration is split up into sections - each Dendrite component has a # configuration section, in addition to the "global" section which applies to # all components. -# -# At a minimum, to get started, you will need to update the settings in the -# "global" section for your deployment, and you will need to check that the -# database "connection_string" line in each component section is correct. -# -# Each component with a "database" section can accept the following formats -# for "connection_string": -# SQLite: file:filename.db -# file:///path/to/filename.db -# PostgreSQL: postgresql://user:pass@hostname/database?params=... -# -# SQLite is embedded into Dendrite and therefore no further prerequisites are -# needed for the database when using SQLite mode. However, performance with -# PostgreSQL is significantly better and recommended for multi-user deployments. -# SQLite is typically around 20-30% slower than PostgreSQL when tested with a -# small number of users and likely will perform worse still with a higher volume -# of users. -# -# The "max_open_conns" and "max_idle_conns" settings configure the maximum -# number of open/idle database connections. The value 0 will use the database -# engine default, and a negative value will use unlimited connections. The -# "conn_max_lifetime" option controls the maximum length of time a database -# connection can be idle in seconds - a negative value is unlimited. # The version of the configuration file. version: 2 @@ -44,9 +21,9 @@ global: # to old signing private keys that were formerly in use on this domain. These # keys will not be used for federation request or event signing, but will be # provided to any other homeserver that asks when trying to verify old events. - # old_private_keys: - # - private_key: old_matrix_key.pem - # expired_at: 1601024554498 + old_private_keys: + # - private_key: old_matrix_key.pem + # expired_at: 1601024554498 # How long a remote server can cache our server signing key before requesting it # again. Increasing this number will reduce the number of requests made by other @@ -54,16 +31,6 @@ global: # considered valid by other homeservers. key_validity_period: 168h0m0s - # Global database connection pool, for PostgreSQL monolith deployments only. If - # this section is populated then you can omit the "database" blocks in all other - # sections. For polylith deployments, or monolith deployments using SQLite databases, - # you must configure the "database" block for each component instead. - # database: - # connection_string: postgres://user:pass@hostname/database?sslmode=disable - # max_open_conns: 100 - # max_idle_conns: 5 - # conn_max_lifetime: -1 - # The server name to delegate server-server communications to, with optional port # e.g. localhost:443 well_known_server_name: "" @@ -74,36 +41,34 @@ global: - matrix.org - vector.im - # Disables federation. Dendrite will not be able to make any outbound HTTP requests - # to other servers and the federation API will not be exposed. + # Disables federation. Dendrite will not be able to communicate with other servers + # in the Matrix federation and the federation API will not be exposed. disable_federation: false - # Configures the handling of presence events. + # Configures the handling of presence events. Inbound controls whether we receive + # presence events from other servers, outbound controls whether we send presence + # events for our local users to other servers. presence: - # Whether inbound presence events are allowed, e.g. receiving presence events from other servers enable_inbound: false - # Whether outbound presence events are allowed, e.g. sending presence events to other servers enable_outbound: false - # Configures opt-in anonymous stats reporting. + # Configures phone-home statistics reporting. These statistics contain the server + # name, number of active users and some information on your deployment config. + # We use this information to understand how Dendrite is being used in the wild. report_stats: - # Whether this instance sends anonymous usage stats enabled: false - - # The endpoint to report the anonymized homeserver usage statistics to. - # Defaults to https://matrix.org/report-usage-stats/push endpoint: https://matrix.org/report-usage-stats/push - # Server notices allows server admins to send messages to all users. + # Server notices allows server admins to send messages to all users on the server. server_notices: enabled: false - # The server localpart to be used when sending notices, ensure this is not yet taken + # The local part, display name and avatar URL (as a mxc:// URL) for the user that + # will send the server notices. These are visible to all users on the deployment. local_part: "_server" - # The displayname to be used when sending notices - display_name: "Server alerts" - # The mxid of the avatar to use + display_name: "Server Alerts" avatar_url: "" - # The roomname to be used when creating messages + # The room name to be used when sending server notices. This room name will + # appear in user clients. room_name: "Server Alerts" # Consent tracking configuration @@ -138,70 +103,57 @@ global: # Configuration for NATS JetStream jetstream: # A list of NATS Server addresses to connect to. If none are specified, an - # internal NATS server will be started automatically when running Dendrite - # in monolith mode. It is required to specify the address of at least one - # NATS Server node if running in polylith mode. + # internal NATS server will be started automatically when running Dendrite in + # monolith mode. For polylith deployments, it is required to specify the address + # of at least one NATS Server node. addresses: - # - localhost:4222 + - hostname:4222 - # Keep all NATS streams in memory, rather than persisting it to the storage - # path below. This option is present primarily for integration testing and - # should not be used on a real world Dendrite deployment. - in_memory: false - - # Persistent directory to store JetStream streams in. This directory - # should be preserved across Dendrite restarts. - storage_path: ./ - - # The prefix to use for stream names for this homeserver - really only - # useful if running more than one Dendrite on the same NATS deployment. + # The prefix to use for stream names for this homeserver - really only useful + # if you are running more than one Dendrite server on the same NATS deployment. topic_prefix: Dendrite # Configuration for Prometheus metric collection. metrics: - # Whether or not Prometheus metrics are enabled. enabled: false - - # HTTP basic authentication to protect access to monitoring. basic_auth: username: metrics password: metrics - # DNS cache options. The DNS cache may reduce the load on DNS servers - # if there is no local caching resolver available for use. + # Optional DNS cache. The DNS cache may reduce the load on DNS servers if there + # is no local caching resolver available for use. dns_cache: - # Whether or not the DNS cache is enabled. enabled: false - - # Maximum number of entries to hold in the DNS cache, and - # for how long those items should be considered valid in seconds. cache_size: 256 - cache_lifetime: "5m" # 5minutes; see https://pkg.go.dev/time@master#ParseDuration for more + cache_lifetime: "5m" # 5 minutes; https://pkg.go.dev/time@master#ParseDuration # Configuration for the Appservice API. app_service_api: internal_api: - listen: http://localhost:7777 # Only used in polylith deployments - connect: http://localhost:7777 # Only used in polylith deployments + listen: http://[::]:7777 # The listen address for incoming API requests + connect: http://app_service_api:7777 # The connect address for other components to use + + # Database configuration for this component. database: - connection_string: file:appservice.db + connection_string: postgresql://username@password:hostname/dendrite_appservice?sslmode=disable max_open_conns: 10 max_idle_conns: 2 conn_max_lifetime: -1 # Disable the validation of TLS certificates of appservices. This is # not recommended in production since it may allow appservice traffic - # to be sent to an unverified endpoint. + # to be sent to an insecure endpoint. disable_tls_validation: false # Appservice configuration files to load into this homeserver. - config_files: [] + config_files: + # - /path/to/appservice_registration.yaml # Configuration for the Client API. client_api: internal_api: - listen: http://localhost:7771 # Only used in polylith deployments - connect: http://localhost:7771 # Only used in polylith deployments + listen: http://[::]:7771 # The listen address for incoming API requests + connect: http://client_api:7771 # The connect address for other components to use external_api: listen: http://[::]:8071 @@ -213,11 +165,13 @@ client_api: # disabled implicitly by setting 'registration_disabled' above. guests_disabled: true - # If set, allows registration by anyone who knows the shared secret, regardless of - # whether registration is otherwise disabled. + # If set, allows registration by anyone who knows the shared secret, regardless + # of whether registration is otherwise disabled. registration_shared_secret: "" - # Whether to require reCAPTCHA for registration. + # Whether to require reCAPTCHA for registration. If you have enabled registration + # then this is HIGHLY RECOMMENDED to reduce the risk of your homeserver being used + # for coordinated spam attacks. enable_registration_captcha: false # Settings for ReCAPTCHA. @@ -229,14 +183,16 @@ client_api: # TURN server information that this homeserver should send to clients. turn: turn_user_lifetime: "" - turn_uris: [] + turn_uris: + # - turn:turn.server.org?transport=udp + # - turn:turn.server.org?transport=tcp turn_shared_secret: "" turn_username: "" turn_password: "" - # Settings for rate-limited endpoints. Rate limiting will kick in after the - # threshold number of "slots" have been taken by requests from a specific - # host. Each "slot" will be released after the cooloff time in milliseconds. + # Settings for rate-limited endpoints. Rate limiting kicks in after the threshold + # number of "slots" have been taken by requests from a specific host. Each "slot" + # will be released after the cooloff time in milliseconds. rate_limiting: enabled: true threshold: 5 @@ -245,18 +201,20 @@ client_api: # Configuration for the Federation API. federation_api: internal_api: - listen: http://localhost:7772 # Only used in polylith deployments - connect: http://localhost:7772 # Only used in polylith deployments + listen: http://[::]:7772 # The listen address for incoming API requests + connect: http://federation_api:7772 # The connect address for other components to use external_api: listen: http://[::]:8072 database: - connection_string: file:federationapi.db + connection_string: postgresql://username@password:hostname/dendrite_federationapi?sslmode=disable max_open_conns: 10 max_idle_conns: 2 conn_max_lifetime: -1 # How many times we will try to resend a failed transaction to a specific server. The - # backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds etc. + # backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds etc. Once + # the max retries are exceeded, Dendrite will no longer try to send transactions to + # that server until it comes back to life and connects to us again. send_max_retries: 16 # Disable the validation of TLS certificates of remote federated homeservers. Do not @@ -282,10 +240,10 @@ federation_api: # Configuration for the Key Server (for end-to-end encryption). key_server: internal_api: - listen: http://localhost:7779 # Only used in polylith deployments - connect: http://localhost:7779 # Only used in polylith deployments + listen: http://[::]:7779 # The listen address for incoming API requests + connect: http://key_server:7779 # The connect address for other components to use database: - connection_string: file:keyserver.db + connection_string: postgresql://username@password:hostname/dendrite_keyserver?sslmode=disable max_open_conns: 10 max_idle_conns: 2 conn_max_lifetime: -1 @@ -293,12 +251,12 @@ key_server: # Configuration for the Media API. media_api: internal_api: - listen: http://localhost:7774 # Only used in polylith deployments - connect: http://localhost:7774 # Only used in polylith deployments + listen: http://[::]:7774 # The listen address for incoming API requests + connect: http://media_api:7774 # The connect address for other components to use external_api: listen: http://[::]:8074 database: - connection_string: file:mediaapi.db + connection_string: postgresql://username@password:hostname/dendrite_mediaapi?sslmode=disable max_open_conns: 5 max_idle_conns: 2 conn_max_lifetime: -1 @@ -307,8 +265,8 @@ media_api: base_path: ./media_store # The maximum allowed file size (in bytes) for media uploads to this homeserver - # (0 = unlimited). If using a reverse proxy, ensure it allows requests at - # least this large (e.g. client_max_body_size in nginx.) + # (0 = unlimited). If using a reverse proxy, ensure it allows requests at least + #this large (e.g. the client_max_body_size setting in nginx). max_file_size_bytes: 10485760 # Whether to dynamically generate thumbnails if needed. @@ -329,15 +287,13 @@ media_api: height: 480 method: scale -# Configuration for experimental MSC's +# Configuration for enabling experimental MSCs on this homeserver. mscs: - # A list of enabled MSC's - # Currently valid values are: - # - msc2836 (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836) - # - msc2946 (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946) - mscs: [] + mscs: + # - msc2836 # (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836) + # - msc2946 # (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946) database: - connection_string: file:mscs.db + connection_string: postgresql://username@password:hostname/dendrite_mscs?sslmode=disable max_open_conns: 5 max_idle_conns: 2 conn_max_lifetime: -1 @@ -345,10 +301,10 @@ mscs: # Configuration for the Room Server. room_server: internal_api: - listen: http://localhost:7770 # Only used in polylith deployments - connect: http://localhost:7770 # Only used in polylith deployments + listen: http://[::]:7770 # The listen address for incoming API requests + connect: http://room_server:7770 # The connect address for other components to use database: - connection_string: file:roomserver.db + connection_string: postgresql://username@password:hostname/dendrite_roomserver?sslmode=disable max_open_conns: 10 max_idle_conns: 2 conn_max_lifetime: -1 @@ -356,12 +312,12 @@ room_server: # Configuration for the Sync API. sync_api: internal_api: - listen: http://localhost:7773 # Only used in polylith deployments - connect: http://localhost:7773 # Only used in polylith deployments + listen: http://[::]:7773 # The listen address for incoming API requests + connect: http://sync_api:7773 # The connect address for other components to use external_api: listen: http://[::]:8073 database: - connection_string: file:syncapi.db + connection_string: postgresql://username@password:hostname/dendrite_syncapi?sslmode=disable max_open_conns: 10 max_idle_conns: 2 conn_max_lifetime: -1 @@ -373,21 +329,23 @@ sync_api: # Configuration for the User API. user_api: - # The cost when hashing passwords on registration/login. Default: 10. Min: 4, Max: 31 - # See https://pkg.go.dev/golang.org/x/crypto/bcrypt for more information. - # Setting this lower makes registration/login consume less CPU resources at the cost of security - # should the database be compromised. Setting this higher makes registration/login consume more - # CPU resources but makes it harder to brute force password hashes. - # This value can be low if performing tests or on embedded Dendrite instances (e.g WASM builds) - # bcrypt_cost: 10 internal_api: - listen: http://localhost:7781 # Only used in polylith deployments - connect: http://localhost:7781 # Only used in polylith deployments + listen: http://[::]:7781 # The listen address for incoming API requests + connect: http://user_api:7781 # The connect address for other components to use account_database: - connection_string: file:userapi_accounts.db + connection_string: postgresql://username@password:hostname/dendrite_userapi?sslmode=disable max_open_conns: 10 max_idle_conns: 2 conn_max_lifetime: -1 + + # The cost when hashing passwords on registration/login. Default: 10. Min: 4, Max: 31 + # See https://pkg.go.dev/golang.org/x/crypto/bcrypt for more information. + # Setting this lower makes registration/login consume less CPU resources at the cost + # of security should the database be compromised. Setting this higher makes registration/login + # consume more CPU resources but makes it harder to brute force password hashes. This value + # can be lowered if performing tests or on embedded Dendrite instances (e.g WASM builds). + bcrypt_cost: 10 + # The length of time that a token issued for a relying party from # /_matrix/client/r0/user/{userId}/openid/request_token endpoint # is considered to be valid in milliseconds. @@ -410,12 +368,13 @@ tracing: baggage_restrictions: null throttler: null -# Logging configuration +# Logging configuration. The "std" logging type controls the logs being sent to +# stdout. The "file" logging type controls logs being written to a log folder on +# the disk. Supported log levels are "debug", "info", "warn", "error". logging: - type: std level: info - type: file - # The logging level, must be one of debug, info, warn, error, fatal, panic. level: info params: path: ./logs diff --git a/docs/CODE_STYLE.md b/docs/CODE_STYLE.md deleted file mode 100644 index 8096ae27c..000000000 --- a/docs/CODE_STYLE.md +++ /dev/null @@ -1,60 +0,0 @@ -# Code Style - -In addition to standard Go code style (`gofmt`, `goimports`), we use `golangci-lint` -to run a number of linters, the exact list can be found under linters in [.golangci.yml](.golangci.yml). -[Installation](https://github.com/golangci/golangci-lint#install-golangci-lint) and [Editor -Integration](https://golangci-lint.run/usage/integrations/#editor-integration) for -it can be found in the readme of golangci-lint. - -For rare cases where a linter is giving a spurious warning, it can be disabled -for that line or statement using a [comment -directive](https://golangci-lint.run/usage/false-positives/#nolint), e.g. `var -bad_name int //nolint:golint,unused`. This should be used sparingly and only -when its clear that the lint warning is spurious. - -The linters can be run using [build/scripts/find-lint.sh](/build/scripts/find-lint.sh) -(see file for docs) or as part of a build/test/lint cycle using -[build/scripts/build-test-lint.sh](/build/scripts/build-test-lint.sh). - - -## Labels - -In addition to `TODO` and `FIXME` we also use `NOTSPEC` to identify deviations -from the Matrix specification. - -## Logging - -We generally prefer to log with static log messages and include any dynamic -information in fields. - -```golang -logger := util.GetLogger(ctx) - -// Not recommended -logger.Infof("Finished processing keys for %s, number of keys %d", name, numKeys) - -// Recommended -logger.WithFields(logrus.Fields{ - "numberOfKeys": numKeys, - "entityName": name, -}).Info("Finished processing keys") -``` - -This is useful when logging to systems that natively understand log fields, as -it allows people to search and process the fields without having to parse the -log message. - - -## Visual Studio Code - -If you use VSCode then the following is an example of a workspace setting that -sets up linting correctly: - -```json -{ - "go.lintTool":"golangci-lint", - "go.lintFlags": [ - "--fast" - ] -} -``` diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 116adfae6..5a89e6841 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -1,55 +1,103 @@ +--- +title: Contributing +parent: Development +permalink: /development/contributing +--- + # Contributing to Dendrite Everyone is welcome to contribute to Dendrite! We aim to make it as easy as possible to get started. -Please ensure that you sign off your contributions! See [Sign Off](#sign-off) -section below. +## Sign off + +We ask that everyone who contributes to the project signs off their contributions +in accordance with the [DCO](https://github.com/matrix-org/matrix-spec/blob/main/CONTRIBUTING.rst#sign-off). +In effect, this means adding a statement to your pull requests or commit messages +along the lines of: + +``` +Signed-off-by: Full Name +``` + +Unfortunately we can't accept contributions without it. ## Getting up and running -See [INSTALL.md](INSTALL.md) for instructions on setting up a running dev -instance of dendrite, and [CODE_STYLE.md](CODE_STYLE.md) for the code style -guide. +See the [Installation](INSTALL.md) section for information on how to build an +instance of Dendrite. You will likely need this in order to test your changes. -We use [golangci-lint](https://github.com/golangci/golangci-lint) to lint -Dendrite which can be executed via: +## Code style +On the whole, the format as prescribed by `gofmt`, `goimports` etc. is exactly +what we use and expect. Please make sure that you run one of these formatters before +submitting your contribution. + +## Comments + +Please make sure that the comments adequately explain *why* your code does what it +does. If there are statements that are not obvious, please comment what they do. + +We also have some special tags which we use for searchability. These are: + +* `// TODO:` for places where a future review, rewrite or refactor is likely required; +* `// FIXME:` for places where we know there is an outstanding bug that needs a fix; +* `// NOTSPEC:` for places where the behaviour specifically does not match what the + [Matrix Specification](https://spec.matrix.org/) prescribes, along with a description + of *why* that is the case. + +## Linting + +We use [golangci-lint](https://github.com/golangci/golangci-lint) to lint Dendrite +which can be executed via: + +```bash +golangci-lint run ``` -$ golangci-lint run -``` + +If you are receiving linter warnings that you are certain are spurious and want to +silence them, you can annotate the relevant lines or methods with a `// nolint:` +comment. Please avoid doing this if you can. + +## Unit tests We also have unit tests which we run via: -``` -$ go test ./... +```bash +go test ./... ``` -## Continuous Integration +In general, we like submissions that come with tests. Anything that proves that the +code is functioning as intended is great, and to ensure that we will find out quickly +in the future if any regressions happen. -When a Pull Request is submitted, continuous integration jobs are run -automatically to ensure the code builds and is relatively well-written. The jobs -are run on [Buildkite](https://buildkite.com/matrix-dot-org/dendrite/), and the -Buildkite pipeline configuration can be found in Matrix.org's [pipelines -repository](https://github.com/matrix-org/pipelines). +We use the standard [Go testing package](https://gobyexample.com/testing) for this, +alongside some helper functions in our own [`test` package](https://pkg.go.dev/github.com/matrix-org/dendrite/test). -If a job fails, click the "details" button and you should be taken to the job's -logs. +## Continuous integration -![Click the details button on the failing build -step](https://raw.githubusercontent.com/matrix-org/dendrite/main/docs/images/details-button-location.jpg) +When a Pull Request is submitted, continuous integration jobs are run automatically +by GitHub actions to ensure that the code builds and works in a number of configurations, +such as different Go versions, using full HTTP APIs and both database engines. +CI will automatically run the unit tests (as above) as well as both of our integration +test suites ([Complement](https://github.com/matrix-org/complement) and +[SyTest](https://github.com/matrix-org/sytest)). -Scroll down to the failing step and you should see some log output. Scan the -logs until you find what it's complaining about, fix it, submit a new commit, -then rinse and repeat until CI passes. +You can see the progress of any CI jobs at the bottom of the Pull Request page, or by +looking at the [Actions](https://github.com/matrix-org/dendrite/actions) tab of the Dendrite +repository. -### Running CI Tests Locally +We generally won't accept a submission unless all of the CI jobs are passing. We +do understand though that sometimes the tests get things wrong — if that's the case, +please also raise a pull request to fix the relevant tests! + +### Running CI tests locally To save waiting for CI to finish after every commit, it is ideal to run the checks locally before pushing, fixing errors first. This also saves other people time as only so many PRs can be tested at a given time. -To execute what Buildkite tests, first run `./build/scripts/build-test-lint.sh`; this +To execute what CI tests, first run `./build/scripts/build-test-lint.sh`; this script will build the code, lint it, and run `go test ./...` with race condition checking enabled. If something needs to be changed, fix it and then run the script again until it no longer complains. Be warned that the linting can take a @@ -64,8 +112,7 @@ passing tests. If these two steps report no problems, the code should be able to pass the CI tests. - -## Picking Things To Do +## Picking things to do If you're new then feel free to pick up an issue labelled [good first issue](https://github.com/matrix-org/dendrite/labels/good%20first%20issue). @@ -81,17 +128,10 @@ We ask people who are familiar with Dendrite to leave the [good first issue](https://github.com/matrix-org/dendrite/labels/good%20first%20issue) issues so that there is always a way for new people to come and get involved. -## Getting Help +## Getting help For questions related to developing on Dendrite we have a dedicated room on Matrix [#dendrite-dev:matrix.org](https://matrix.to/#/#dendrite-dev:matrix.org) where we're happy to help. -For more general questions please use -[#dendrite:matrix.org](https://matrix.to/#/#dendrite:matrix.org). - -## Sign off - -We ask that everyone who contributes to the project signs off their -contributions, in accordance with the -[DCO](https://github.com/matrix-org/matrix-spec/blob/main/CONTRIBUTING.rst#sign-off). +For more general questions please use [#dendrite:matrix.org](https://matrix.to/#/#dendrite:matrix.org). diff --git a/docs/DESIGN.md b/docs/DESIGN.md deleted file mode 100644 index 80e251c5e..000000000 --- a/docs/DESIGN.md +++ /dev/null @@ -1,140 +0,0 @@ -# Design - -## Log Based Architecture - -### Decomposition and Decoupling - -A matrix homeserver can be built around append-only event logs built from the -messages, receipts, presence, typing notifications, device messages and other -events sent by users on the homeservers or by other homeservers. - -The server would then decompose into two categories: writers that add new -entries to the logs and readers that read those entries. - -The event logs then serve to decouple the two components, the writers and -readers need only agree on the format of the entries in the event log. -This format could be largely derived from the wire format of the events used -in the client and federation protocols: - - - C-S API +---------+ Event Log +---------+ C-S API - ---------> | |+ (e.g. kafka) | |+ ---------> - | Writers || =============> | Readers || - ---------> | || | || ---------> - S-S API +---------+| +---------+| S-S API - +---------+ +---------+ - -However the way matrix handles state events in a room creates a few -complications for this model. - - 1) Writers require the room state at an event to check if it is allowed. - 2) Readers require the room state at an event to determine the users and - servers that are allowed to see the event. - 3) A client can query the current state of the room from a reader. - -The writers and readers cannot extract the necessary information directly from -the event logs because it would take too long to extract the information as the -state is built up by collecting individual state events from the event history. - -The writers and readers therefore need access to something that stores copies -of the event state in a form that can be efficiently queried. One possibility -would be for the readers and writers to maintain copies of the current state -in local databases. A second possibility would be to add a dedicated component -that maintained the state of the room and exposed an API that the readers and -writers could query to get the state. The second has the advantage that the -state is calculated and stored in a single location. - - - C-S API +---------+ Log +--------+ Log +---------+ C-S API - ---------> | |+ ======> | | ======> | |+ ---------> - | Writers || | Room | | Readers || - ---------> | || <------ | Server | ------> | || ---------> - S-S API +---------+| Query | | Query +---------+| S-S API - +---------+ +--------+ +---------+ - - -The room server can annotate the events it logs to the readers with room state -so that the readers can avoid querying the room server unnecessarily. - -[This architecture can be extended to cover most of the APIs.](WIRING.md) - -## How things are supposed to work. - -### Local client sends an event in an existing room. - - 0) The client sends a PUT `/_matrix/client/r0/rooms/{roomId}/send` request - and an HTTP loadbalancer routes the request to a ClientAPI. - - 1) The ClientAPI: - - * Authenticates the local user using the `access_token` sent in the HTTP - request. - * Checks if it has already processed or is processing a request with the - same `txnID`. - * Calculates which state events are needed to auth the request. - * Queries the necessary state events and the latest events in the room - from the RoomServer. - * Confirms that the room exists and checks whether the event is allowed by - the auth checks. - * Builds and signs the events. - * Writes the event to a "InputRoomEvent" kafka topic. - * Send a `200 OK` response to the client. - - 2) The RoomServer reads the event from "InputRoomEvent" kafka topic: - - * Checks if it has already has a copy of the event. - * Checks if the event is allowed by the auth checks using the auth events - at the event. - * Calculates the room state at the event. - * Works out what the latest events in the room after processing this event - are. - * Calculate how the changes in the latest events affect the current state - of the room. - * TODO: Workout what events determine the visibility of this event to other - users - * Writes the event along with the changes in current state to an - "OutputRoomEvent" kafka topic. It writes all the events for a room to - the same kafka partition. - - 3a) The ClientSync reads the event from the "OutputRoomEvent" kafka topic: - - * Updates its copy of the current state for the room. - * Works out which users need to be notified about the event. - * Wakes up any pending `/_matrix/client/r0/sync` requests for those users. - * Adds the event to the recent timeline events for the room. - - 3b) The FederationSender reads the event from the "OutputRoomEvent" kafka topic: - - * Updates its copy of the current state for the room. - * Works out which remote servers need to be notified about the event. - * Sends a `/_matrix/federation/v1/send` request to those servers. - * Or if there is a request in progress then add the event to a queue to be - sent when the previous request finishes. - -### Remote server sends an event in an existing room. - - 0) The remote server sends a `PUT /_matrix/federation/v1/send` request and an - HTTP loadbalancer routes the request to a FederationReceiver. - - 1) The FederationReceiver: - - * Authenticates the remote server using the "X-Matrix" authorisation header. - * Checks if it has already processed or is processing a request with the - same `txnID`. - * Checks the signatures for the events. - Fetches the ed25519 keys for the event senders if necessary. - * Queries the RoomServer for a copy of the state of the room at each event. - * If the RoomServer doesn't know the state of the room at an event then - query the state of the room at the event from the remote server using - `GET /_matrix/federation/v1/state_ids` falling back to - `GET /_matrix/federation/v1/state` if necessary. - * Once the state at each event is known check whether the events are - allowed by the auth checks against the state at each event. - * For each event that is allowed write the event to the "InputRoomEvent" - kafka topic. - * Send a 200 OK response to the remote server listing which events were - successfully processed and which events failed - - 2) The RoomServer processes the event the same as it would a local event. - - 3a) The ClientSync processes the event the same as it would a local event. diff --git a/docs/FAQ.md b/docs/FAQ.md index 47eaecf0f..47f39b9e6 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -1,26 +1,34 @@ -# Frequently Asked Questions +--- +title: FAQ +nav_order: 1 +permalink: /faq +--- -### Is Dendrite stable? +# FAQ + +## Is Dendrite stable? Mostly, although there are still bugs and missing features. If you are a confident power user and you are happy to spend some time debugging things when they go wrong, then please try out Dendrite. If you are a community, organisation or business that demands stability and uptime, then Dendrite is not for you yet - please install Synapse instead. -### Is Dendrite feature-complete? +## Is Dendrite feature-complete? No, although a good portion of the Matrix specification has been implemented. Mostly missing are client features - see the readme at the root of the repository for more information. -### Is there a migration path from Synapse to Dendrite? +## Is there a migration path from Synapse to Dendrite? -No, not at present. There will be in the future when Dendrite reaches version 1.0. +No, not at present. There will be in the future when Dendrite reaches version 1.0. For now it is not +possible to migrate an existing Synapse deployment to Dendrite. -### Can I use Dendrite with an existing Synapse database? +## Can I use Dendrite with an existing Synapse database? No, Dendrite has a very different database schema to Synapse and the two are not interchangeable. -### Should I run a monolith or a polylith deployment? +## Should I run a monolith or a polylith deployment? -Monolith deployments are always preferred where possible, and at this time, are far better tested than polylith deployments are. The only reason to consider a polylith deployment is if you wish to run different Dendrite components on separate physical machines. +Monolith deployments are always preferred where possible, and at this time, are far better tested than polylith deployments are. The only reason to consider a polylith deployment is if you wish to run different Dendrite components on separate physical machines, but this is an advanced configuration which we don't +recommend. -### I've installed Dendrite but federation isn't working +## I've installed Dendrite but federation isn't working Check the [Federation Tester](https://federationtester.matrix.org). You need at least: @@ -28,54 +36,57 @@ Check the [Federation Tester](https://federationtester.matrix.org). You need at * A valid TLS certificate for that DNS name * Either DNS SRV records or well-known files -### Does Dendrite work with my favourite client? +## Does Dendrite work with my favourite client? It should do, although we are aware of some minor issues: * **Element Android**: registration does not work, but logging in with an existing account does * **Hydrogen**: occasionally sync can fail due to gaps in the `since` parameter, but clearing the cache fixes this -### Does Dendrite support push notifications? +## Does Dendrite support push notifications? Yes, we have experimental support for push notifications. Configure them in the usual way in your Matrix client. -### Does Dendrite support application services/bridges? +## Does Dendrite support application services/bridges? Possibly - Dendrite does have some application service support but it is not well tested. Please let us know by raising a GitHub issue if you try it and run into problems. Bridges known to work (as of v0.5.1): -- [Telegram](https://docs.mau.fi/bridges/python/telegram/index.html) -- [WhatsApp](https://docs.mau.fi/bridges/go/whatsapp/index.html) -- [Signal](https://docs.mau.fi/bridges/python/signal/index.html) -- [probably all other mautrix bridges](https://docs.mau.fi/bridges/) -Remember to add the config file(s) to the `app_service_api` [config](https://github.com/matrix-org/dendrite/blob/de38be469a23813921d01bef3e14e95faab2a59e/dendrite-config.yaml#L130-L131). +* [Telegram](https://docs.mau.fi/bridges/python/telegram/index.html) +* [WhatsApp](https://docs.mau.fi/bridges/go/whatsapp/index.html) +* [Signal](https://docs.mau.fi/bridges/python/signal/index.html) +* [probably all other mautrix bridges](https://docs.mau.fi/bridges/) -### Is it possible to prevent communication with the outside world? +Remember to add the config file(s) to the `app_service_api` section of the config file. -Yes, you can do this by disabling federation - set `disable_federation` to `true` in the `global` section of the Dendrite configuration file. +## Is it possible to prevent communication with the outside world? -### Should I use PostgreSQL or SQLite for my databases? +Yes, you can do this by disabling federation - set `disable_federation` to `true` in the `global` section of the Dendrite configuration file. -Please use PostgreSQL wherever possible, especially if you are planning to run a homeserver that caters to more than a couple of users. +## Should I use PostgreSQL or SQLite for my databases? -### Dendrite is using a lot of CPU +Please use PostgreSQL wherever possible, especially if you are planning to run a homeserver that caters to more than a couple of users. -Generally speaking, you should expect to see some CPU spikes, particularly if you are joining or participating in large rooms. However, constant/sustained high CPU usage is not expected - if you are experiencing that, please join `#dendrite-dev:matrix.org` and let us know, or file a GitHub issue. +## Dendrite is using a lot of CPU -### Dendrite is using a lot of RAM +Generally speaking, you should expect to see some CPU spikes, particularly if you are joining or participating in large rooms. However, constant/sustained high CPU usage is not expected - if you are experiencing that, please join `#dendrite-dev:matrix.org` and let us know what you were doing when the +CPU usage shot up, or file a GitHub issue. If you can take a [CPU profile](PROFILING.md) then that would +be a huge help too, as that will help us to understand where the CPU time is going. -A lot of users report that Dendrite is using a lot of RAM, sometimes even gigabytes of it. This is usually due to Go's allocator behaviour, which tries to hold onto allocated memory until the operating system wants to reclaim it for something else. This can make the memory usage look significantly inflated in tools like `top`/`htop` when actually most of that memory is not really in use at all. +## Dendrite is using a lot of RAM -If you want to prevent this behaviour so that the Go runtime releases memory normally, start Dendrite using the `GODEBUG=madvdontneed=1` environment variable. It is also expected that the allocator behaviour will be changed again in Go 1.16 so that it does not hold onto memory unnecessarily in this way. +As above with CPU usage, some memory spikes are expected if Dendrite is doing particularly heavy work +at a given instant. However, if it is using more RAM than you expect for a long time, that's probably +not expected. Join `#dendrite-dev:matrix.org` and let us know what you were doing when the memory usage +ballooned, or file a GitHub issue if you can. If you can take a [memory profile](PROFILING.md) then that +would be a huge help too, as that will help us to understand where the memory usage is happening. -If you are running with `GODEBUG=madvdontneed=1` and still see hugely inflated memory usage then that's quite possibly a bug - please join `#dendrite-dev:matrix.org` and let us know, or file a GitHub issue. - -### Dendrite is running out of PostgreSQL database connections +## Dendrite is running out of PostgreSQL database connections You may need to revisit the connection limit of your PostgreSQL server and/or make changes to the `max_connections` lines in your Dendrite configuration. Be aware that each Dendrite component opens its own database connections and has its own connection limit, even in monolith mode! -### What is being reported when enabling anonymous stats? +## What is being reported when enabling anonymous stats? If anonymous stats reporting is enabled, the following data is send to the defined endpoint. @@ -116,4 +127,4 @@ If anonymous stats reporting is enabled, the following data is send to the defin "uptime_seconds": 30, "version": "0.8.2" } -``` \ No newline at end of file +``` diff --git a/docs/Gemfile b/docs/Gemfile new file mode 100644 index 000000000..a6aa152a2 --- /dev/null +++ b/docs/Gemfile @@ -0,0 +1,5 @@ +source "https://rubygems.org" +gem "github-pages", "~> 226", group: :jekyll_plugins +group :jekyll_plugins do + gem "jekyll-feed", "~> 0.15.1" +end diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock new file mode 100644 index 000000000..e62aa4ce3 --- /dev/null +++ b/docs/Gemfile.lock @@ -0,0 +1,283 @@ +GEM + remote: https://rubygems.org/ + specs: + activesupport (6.0.5) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 0.7, < 2) + minitest (~> 5.1) + tzinfo (~> 1.1) + zeitwerk (~> 2.2, >= 2.2.2) + addressable (2.8.0) + public_suffix (>= 2.0.2, < 5.0) + coffee-script (2.4.1) + coffee-script-source + execjs + coffee-script-source (1.11.1) + colorator (1.1.0) + commonmarker (0.23.4) + concurrent-ruby (1.1.10) + dnsruby (1.61.9) + simpleidn (~> 0.1) + em-websocket (0.5.3) + eventmachine (>= 0.12.9) + http_parser.rb (~> 0) + ethon (0.15.0) + ffi (>= 1.15.0) + eventmachine (1.2.7) + execjs (2.8.1) + faraday (1.10.0) + faraday-em_http (~> 1.0) + faraday-em_synchrony (~> 1.0) + faraday-excon (~> 1.1) + faraday-httpclient (~> 1.0) + faraday-multipart (~> 1.0) + faraday-net_http (~> 1.0) + faraday-net_http_persistent (~> 1.0) + faraday-patron (~> 1.0) + faraday-rack (~> 1.0) + faraday-retry (~> 1.0) + ruby2_keywords (>= 0.0.4) + faraday-em_http (1.0.0) + faraday-em_synchrony (1.0.0) + faraday-excon (1.1.0) + faraday-httpclient (1.0.1) + faraday-multipart (1.0.3) + multipart-post (>= 1.2, < 3) + faraday-net_http (1.0.1) + faraday-net_http_persistent (1.2.0) + faraday-patron (1.0.0) + faraday-rack (1.0.0) + faraday-retry (1.0.3) + ffi (1.15.5) + forwardable-extended (2.6.0) + gemoji (3.0.1) + github-pages (226) + github-pages-health-check (= 1.17.9) + jekyll (= 3.9.2) + jekyll-avatar (= 0.7.0) + jekyll-coffeescript (= 1.1.1) + jekyll-commonmark-ghpages (= 0.2.0) + jekyll-default-layout (= 0.1.4) + jekyll-feed (= 0.15.1) + jekyll-gist (= 1.5.0) + jekyll-github-metadata (= 2.13.0) + jekyll-include-cache (= 0.2.1) + jekyll-mentions (= 1.6.0) + jekyll-optional-front-matter (= 0.3.2) + jekyll-paginate (= 1.1.0) + jekyll-readme-index (= 0.3.0) + jekyll-redirect-from (= 0.16.0) + jekyll-relative-links (= 0.6.1) + jekyll-remote-theme (= 0.4.3) + jekyll-sass-converter (= 1.5.2) + jekyll-seo-tag (= 2.8.0) + jekyll-sitemap (= 1.4.0) + jekyll-swiss (= 1.0.0) + jekyll-theme-architect (= 0.2.0) + jekyll-theme-cayman (= 0.2.0) + jekyll-theme-dinky (= 0.2.0) + jekyll-theme-hacker (= 0.2.0) + jekyll-theme-leap-day (= 0.2.0) + jekyll-theme-merlot (= 0.2.0) + jekyll-theme-midnight (= 0.2.0) + jekyll-theme-minimal (= 0.2.0) + jekyll-theme-modernist (= 0.2.0) + jekyll-theme-primer (= 0.6.0) + jekyll-theme-slate (= 0.2.0) + jekyll-theme-tactile (= 0.2.0) + jekyll-theme-time-machine (= 0.2.0) + jekyll-titles-from-headings (= 0.5.3) + jemoji (= 0.12.0) + kramdown (= 2.3.2) + kramdown-parser-gfm (= 1.1.0) + liquid (= 4.0.3) + mercenary (~> 0.3) + minima (= 2.5.1) + nokogiri (>= 1.13.4, < 2.0) + rouge (= 3.26.0) + terminal-table (~> 1.4) + github-pages-health-check (1.17.9) + addressable (~> 2.3) + dnsruby (~> 1.60) + octokit (~> 4.0) + public_suffix (>= 3.0, < 5.0) + typhoeus (~> 1.3) + html-pipeline (2.14.1) + activesupport (>= 2) + nokogiri (>= 1.4) + http_parser.rb (0.8.0) + i18n (0.9.5) + concurrent-ruby (~> 1.0) + jekyll (3.9.2) + addressable (~> 2.4) + colorator (~> 1.0) + em-websocket (~> 0.5) + i18n (~> 0.7) + jekyll-sass-converter (~> 1.0) + jekyll-watch (~> 2.0) + kramdown (>= 1.17, < 3) + liquid (~> 4.0) + mercenary (~> 0.3.3) + pathutil (~> 0.9) + rouge (>= 1.7, < 4) + safe_yaml (~> 1.0) + jekyll-avatar (0.7.0) + jekyll (>= 3.0, < 5.0) + jekyll-coffeescript (1.1.1) + coffee-script (~> 2.2) + coffee-script-source (~> 1.11.1) + jekyll-commonmark (1.4.0) + commonmarker (~> 0.22) + jekyll-commonmark-ghpages (0.2.0) + commonmarker (~> 0.23.4) + jekyll (~> 3.9.0) + jekyll-commonmark (~> 1.4.0) + rouge (>= 2.0, < 4.0) + jekyll-default-layout (0.1.4) + jekyll (~> 3.0) + jekyll-feed (0.15.1) + jekyll (>= 3.7, < 5.0) + jekyll-gist (1.5.0) + octokit (~> 4.2) + jekyll-github-metadata (2.13.0) + jekyll (>= 3.4, < 5.0) + octokit (~> 4.0, != 4.4.0) + jekyll-include-cache (0.2.1) + jekyll (>= 3.7, < 5.0) + jekyll-mentions (1.6.0) + html-pipeline (~> 2.3) + jekyll (>= 3.7, < 5.0) + jekyll-optional-front-matter (0.3.2) + jekyll (>= 3.0, < 5.0) + jekyll-paginate (1.1.0) + jekyll-readme-index (0.3.0) + jekyll (>= 3.0, < 5.0) + jekyll-redirect-from (0.16.0) + jekyll (>= 3.3, < 5.0) + jekyll-relative-links (0.6.1) + jekyll (>= 3.3, < 5.0) + jekyll-remote-theme (0.4.3) + addressable (~> 2.0) + jekyll (>= 3.5, < 5.0) + jekyll-sass-converter (>= 1.0, <= 3.0.0, != 2.0.0) + rubyzip (>= 1.3.0, < 3.0) + jekyll-sass-converter (1.5.2) + sass (~> 3.4) + jekyll-seo-tag (2.8.0) + jekyll (>= 3.8, < 5.0) + jekyll-sitemap (1.4.0) + jekyll (>= 3.7, < 5.0) + jekyll-swiss (1.0.0) + jekyll-theme-architect (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-cayman (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-dinky (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-hacker (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-leap-day (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-merlot (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-midnight (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-minimal (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-modernist (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-primer (0.6.0) + jekyll (> 3.5, < 5.0) + jekyll-github-metadata (~> 2.9) + jekyll-seo-tag (~> 2.0) + jekyll-theme-slate (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-tactile (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-time-machine (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-titles-from-headings (0.5.3) + jekyll (>= 3.3, < 5.0) + jekyll-watch (2.2.1) + listen (~> 3.0) + jemoji (0.12.0) + gemoji (~> 3.0) + html-pipeline (~> 2.2) + jekyll (>= 3.0, < 5.0) + kramdown (2.3.2) + rexml + kramdown-parser-gfm (1.1.0) + kramdown (~> 2.0) + liquid (4.0.3) + listen (3.7.1) + rb-fsevent (~> 0.10, >= 0.10.3) + rb-inotify (~> 0.9, >= 0.9.10) + mercenary (0.3.6) + minima (2.5.1) + jekyll (>= 3.5, < 5.0) + jekyll-feed (~> 0.9) + jekyll-seo-tag (~> 2.1) + minitest (5.15.0) + multipart-post (2.1.1) + nokogiri (1.13.6-arm64-darwin) + racc (~> 1.4) + octokit (4.22.0) + faraday (>= 0.9) + sawyer (~> 0.8.0, >= 0.5.3) + pathutil (0.16.2) + forwardable-extended (~> 2.6) + public_suffix (4.0.7) + racc (1.6.0) + rb-fsevent (0.11.1) + rb-inotify (0.10.1) + ffi (~> 1.0) + rexml (3.2.5) + rouge (3.26.0) + ruby2_keywords (0.0.5) + rubyzip (2.3.2) + safe_yaml (1.0.5) + sass (3.7.4) + sass-listen (~> 4.0.0) + sass-listen (4.0.0) + rb-fsevent (~> 0.9, >= 0.9.4) + rb-inotify (~> 0.9, >= 0.9.7) + sawyer (0.8.2) + addressable (>= 2.3.5) + faraday (> 0.8, < 2.0) + simpleidn (0.2.1) + unf (~> 0.1.4) + terminal-table (1.8.0) + unicode-display_width (~> 1.1, >= 1.1.1) + thread_safe (0.3.6) + typhoeus (1.4.0) + ethon (>= 0.9.0) + tzinfo (1.2.9) + thread_safe (~> 0.1) + unf (0.1.4) + unf_ext + unf_ext (0.0.8.1) + unicode-display_width (1.8.0) + zeitwerk (2.5.4) + +PLATFORMS + arm64-darwin-21 + +DEPENDENCIES + github-pages (~> 226) + jekyll-feed (~> 0.15.1) + minima (~> 2.5.1) + +BUNDLED WITH + 2.3.7 diff --git a/docs/INSTALL.md b/docs/INSTALL.md index ca1316aca..add822108 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -1,283 +1,15 @@ -# Installing Dendrite - -Dendrite can be run in one of two configurations: - -* **Monolith mode**: All components run in the same process. In this mode, - it is possible to run an in-process [NATS Server](https://github.com/nats-io/nats-server) - instead of running a standalone deployment. This will usually be the preferred model for - low-to-mid volume deployments, providing the best balance between performance and resource usage. - -* **Polylith mode**: A cluster of individual components running in their own processes, dealing - with different aspects of the Matrix protocol (see [WIRING.md](WIRING-Current.md)). Components - communicate with each other using internal HTTP APIs and [NATS Server](https://github.com/nats-io/nats-server). - This will almost certainly be the preferred model for very large deployments but scalability - comes with a cost. API calls are expensive and therefore a polylith deployment may end up using - disproportionately more resources for a smaller number of users compared to a monolith deployment. - -In almost all cases, it is **recommended to run in monolith mode with PostgreSQL databases**. - -Regardless of whether you are running in polylith or monolith mode, each Dendrite component that -requires storage has its own database connections. Both Postgres and SQLite are supported and can -be mixed-and-matched across components as needed in the configuration file. - -Be advised that Dendrite is still in development and it's not recommended for -use in production environments just yet! - -## Requirements - -Dendrite requires: - -* Go 1.16 or higher -* PostgreSQL 12 or higher (if using PostgreSQL databases, not needed for SQLite) - -If you want to run a polylith deployment, you also need: - -* A standalone [NATS Server](https://github.com/nats-io/nats-server) deployment with JetStream enabled - -If you want to build it on Windows, you need `gcc` in the path: - -* [MinGW-w64](https://www.mingw-w64.org/) - -## Building Dendrite - -Start by cloning the code: - -```bash -git clone https://github.com/matrix-org/dendrite -cd dendrite -``` - -Then build it: - -* Linux or UNIX-like systems: - ```bash - ./build.sh - ``` - -* Windows: - ```dos - build.cmd - ``` - -## Install NATS Server - -Follow the [NATS Server installation instructions](https://docs.nats.io/running-a-nats-service/introduction/installation) and then [start your NATS deployment](https://docs.nats.io/running-a-nats-service/introduction/running). - -JetStream must be enabled, either by passing the `-js` flag to `nats-server`, -or by specifying the `store_dir` option in the the `jetstream` configuration. - -## Configuration - -### PostgreSQL database setup - -Assuming that PostgreSQL 12 (or later) is installed: - -* Create role, choosing a new password when prompted: - - ```bash - sudo -u postgres createuser -P dendrite - ``` - -At this point you have a choice on whether to run all of the Dendrite -components from a single database, or for each component to have its -own database. For most deployments, running from a single database will -be sufficient, although you may wish to separate them if you plan to -split out the databases across multiple machines in the future. - -On macOS, omit `sudo -u postgres` from the below commands. - -* If you want to run all Dendrite components from a single database: - - ```bash - sudo -u postgres createdb -O dendrite dendrite - ``` - - ... in which case your connection string will look like `postgres://user:pass@database/dendrite`. - -* If you want to run each Dendrite component with its own database: - - ```bash - for i in mediaapi syncapi roomserver federationapi appservice keyserver userapi_accounts; do - sudo -u postgres createdb -O dendrite dendrite_$i - done - ``` - - ... in which case your connection string will look like `postgres://user:pass@database/dendrite_componentname`. - -### SQLite database setup - -**WARNING:** SQLite is suitable for small experimental deployments only and should not be used in production - use PostgreSQL instead for any user-facing federating installation! - -Dendrite can use the built-in SQLite database engine for small setups. -The SQLite databases do not need to be pre-built - Dendrite will -create them automatically at startup. - -### Server key generation - -Each Dendrite installation requires: - -* A unique Matrix signing private key -* A valid and trusted TLS certificate and private key - -To generate a Matrix signing private key: - -```bash -./bin/generate-keys --private-key matrix_key.pem -``` - -**WARNING:** Make sure take a safe backup of this key! You will likely need it if you want to reinstall Dendrite, or -any other Matrix homeserver, on the same domain name in the future. If you lose this key, you may have trouble joining -federated rooms. - -For testing, you can generate a self-signed certificate and key, although this will not work for public federation: - -```bash -./bin/generate-keys --tls-cert server.crt --tls-key server.key -``` - -If you have server keys from an older Synapse instance, -[convert them](serverkeyformat.md#converting-synapse-keys) to Dendrite's PEM -format and configure them as `old_private_keys` in your config. - -### Configuration file - -Create config file, based on `dendrite-config.yaml`. Call it `dendrite.yaml`. Things that will need editing include *at least*: - -* The `server_name` entry to reflect the hostname of your Dendrite server -* The `database` lines with an updated connection string based on your - desired setup, e.g. replacing `database` with the name of the database: - * For Postgres: `postgres://dendrite:password@localhost/database`, e.g. - * `postgres://dendrite:password@localhost/dendrite_userapi_account` to connect to PostgreSQL with SSL/TLS - * `postgres://dendrite:password@localhost/dendrite_userapi_account?sslmode=disable` to connect to PostgreSQL without SSL/TLS - * For SQLite on disk: `file:component.db` or `file:///path/to/component.db`, e.g. `file:userapi_account.db` - * Postgres and SQLite can be mixed and matched on different components as desired. -* Either one of the following in the `jetstream` configuration section: - * The `addresses` option — a list of one or more addresses of an external standalone - NATS Server deployment - * The `storage_path` — where on the filesystem the built-in NATS server should - store durable queues, if using the built-in NATS server - -There are other options which may be useful so review them all. In particular, -if you are trying to federate from your Dendrite instance into public rooms -then configuring `key_perspectives` (like `matrix.org` in the sample) can -help to improve reliability considerably by allowing your homeserver to fetch -public keys for dead homeservers from somewhere else. - -**WARNING:** Dendrite supports running all components from the same database in -PostgreSQL mode, but this is **NOT** a supported configuration with SQLite. When -using SQLite, all components **MUST** use their own database file. - -## Starting a monolith server - -The monolith server can be started as shown below. By default it listens for -HTTP connections on port 8008, so you can configure your Matrix client to use -`http://servername:8008` as the server: - -```bash -./bin/dendrite-monolith-server -``` - -If you set `--tls-cert` and `--tls-key` as shown below, it will also listen -for HTTPS connections on port 8448: - -```bash -./bin/dendrite-monolith-server --tls-cert=server.crt --tls-key=server.key -``` - -If the `jetstream` section of the configuration contains no `addresses` but does -contain a `store_dir`, Dendrite will start up a built-in NATS JetStream node -automatically, eliminating the need to run a separate NATS server. - -## Starting a polylith deployment - -The following contains scripts which will run all the required processes in order to point a Matrix client at Dendrite. - -### nginx (or other reverse proxy) - -This is what your clients and federated hosts will talk to. It must forward -requests onto the correct API server based on URL: - -* `/_matrix/client` to the client API server -* `/_matrix/federation` to the federation API server -* `/_matrix/key` to the federation API server -* `/_matrix/media` to the media API server - -See `docs/nginx/polylith-sample.conf` for a sample configuration. - -### Client API server - -This is what implements CS API endpoints. Clients talk to this via the proxy in -order to send messages, create and join rooms, etc. - -```bash -./bin/dendrite-polylith-multi --config=dendrite.yaml clientapi -``` - -### Sync server - -This is what implements `/sync` requests. Clients talk to this via the proxy -in order to receive messages. - -```bash -./bin/dendrite-polylith-multi --config=dendrite.yaml syncapi -``` - -### Media server - -This implements `/media` requests. Clients talk to this via the proxy in -order to upload and retrieve media. - -```bash -./bin/dendrite-polylith-multi --config=dendrite.yaml mediaapi -``` - -### Federation API server - -This implements the federation API. Servers talk to this via the proxy in -order to send transactions. This is only required if you want to support -federation. - -```bash -./bin/dendrite-polylith-multi --config=dendrite.yaml federationapi -``` - -### Internal components - -This refers to components that are not directly spoken to by clients. They are only -contacted by other components. This includes the following components. - -#### Room server - -This is what implements the room DAG. Clients do not talk to this. - -```bash -./bin/dendrite-polylith-multi --config=dendrite.yaml roomserver -``` - -#### Appservice server - -This sends events from the network to [application -services](https://matrix.org/docs/spec/application_service/unstable.html) -running locally. This is only required if you want to support running -application services on your homeserver. - -```bash -./bin/dendrite-polylith-multi --config=dendrite.yaml appservice -``` - -#### Key server - -This manages end-to-end encryption keys for users. - -```bash -./bin/dendrite-polylith-multi --config=dendrite.yaml keyserver -``` - -#### User server - -This manages user accounts, device access tokens and user account data, -amongst other things. - -```bash -./bin/dendrite-polylith-multi --config=dendrite.yaml userapi -``` +# Installation + +Please note that new installation instructions can be found +on the [new documentation site](https://matrix-org.github.io/dendrite/), +or alternatively, in the [installation](installation/) folder: + +1. [Planning your deployment](installation/1_planning.md) +2. [Setting up the domain](installation/2_domainname.md) +3. [Preparing database storage](installation/3_database.md) +4. [Generating signing keys](installation/4_signingkey.md) +5. [Installing as a monolith](installation/5_install_monolith.md) +6. [Installing as a polylith](installation/6_install_polylith.md) +7. [Populate the configuration](installation/7_configuration.md) +8. [Starting the monolith](installation/8_starting_monolith.md) +9. [Starting the polylith](installation/9_starting_polylith.md) diff --git a/docs/PROFILING.md b/docs/PROFILING.md index b026a8aed..f3b573472 100644 --- a/docs/PROFILING.md +++ b/docs/PROFILING.md @@ -1,8 +1,14 @@ +--- +title: Profiling +parent: Development +permalink: /development/profiling +--- + # Profiling Dendrite If you are running into problems with Dendrite using excessive resources (e.g. CPU or RAM) then you can use the profiler to work out what is happening. -Dendrite contains an embedded profiler called `pprof`, which is a part of the standard Go toolchain. +Dendrite contains an embedded profiler called `pprof`, which is a part of the standard Go toolchain. ## Enable the profiler @@ -16,7 +22,7 @@ If pprof has been enabled successfully, a log line at startup will show that ppr ``` WARN[2020-12-03T13:32:33.669405000Z] [/Users/neilalexander/Desktop/dendrite/internal/log.go:87] SetupPprof - Starting pprof on localhost:65432 + Starting pprof on localhost:65432 ``` All examples from this point forward assume `PPROFLISTEN=localhost:65432` but you may need to adjust as necessary for your setup. diff --git a/docs/WIRING-Current.md b/docs/WIRING-Current.md deleted file mode 100644 index b74f341e5..000000000 --- a/docs/WIRING-Current.md +++ /dev/null @@ -1,71 +0,0 @@ -This document details how various components communicate with each other. There are two kinds of components: - - Public-facing: exposes CS/SS API endpoints and need to be routed to via client-api-proxy or equivalent. - - Internal-only: exposes internal APIs and produces Kafka events. - -## Internal HTTP APIs - -Not everything can be done using Kafka logs. For example, requesting the latest events in a room is much better suited to -a request/response model like HTTP or RPC. Therefore, components can expose "internal APIs" which sit outside of Kafka logs. -Note in Monolith mode these are actually direct function calls and are not serialised HTTP requests. - -``` - Tier 1 Sync FederationAPI ClientAPI MediaAPI -Public Facing | | | | | | | | | | - 2 .-------3-----------------` | | | `--------|-|-|-|--11--------------------. - | | .--------4----------------------------------` | | | | - | | | .---5-----------` | | | | | | - | | | | .---6----------------------------` | | | - | | | | | | .-----7----------` | | - | | | | | 8 | | 10 | - | | | | | | | `---9----. | | - V V V V V V V V V V - Tier 2 Roomserver EDUServer FedSender AppService KeyServer ServerKeyAPI -Internal only | `------------------------12----------^ ^ - `------------------------------------------------------------13----------` - - Client ---> Server -``` -- 2 (Sync -> Roomserver): When making backfill requests -- 3 (FedAPI -> Roomserver): Calculating (prev/auth events) and sending new events, processing backfill/state/state_ids requests -- 4 (ClientAPI -> Roomserver): Calculating (prev/auth events) and sending new events, processing /state requests -- 5 (FedAPI -> EDUServer): Sending typing/send-to-device events -- 6 (ClientAPI -> EDUServer): Sending typing/send-to-device events -- 7 (ClientAPI -> FedSender): Handling directory lookups -- 8 (FedAPI -> FedSender): Resetting backoffs when receiving traffic from a server. Querying joined hosts when handling alias lookup requests -- 9 (FedAPI -> AppService): Working out if the client is an appservice user -- 10 (ClientAPI -> AppService): Working out if the client is an appservice user -- 11 (FedAPI -> ServerKeyAPI): Verifying incoming event signatures -- 12 (FedSender -> ServerKeyAPI): Verifying event signatures of responses (e.g from send_join) -- 13 (Roomserver -> ServerKeyAPI): Verifying event signatures of backfilled events - -In addition to this, all public facing components (Tier 1) talk to the `UserAPI` to verify access tokens and extract profile information where needed. - -## Kafka logs - -``` - .----1--------------------------------------------. - V | - Tier 1 Sync FederationAPI ClientAPI MediaAPI -Public Facing ^ ^ ^ - | | | - 2 | | - | `-3------------. | - | | | - | | | - | | | - | .--------4-----|------------------------------` - | | | - Tier 2 Roomserver EDUServer FedSender AppService KeyServer ServerKeyAPI -Internal only | | ^ ^ - | `-----5----------` | - `--------------------6--------` - - -Producer ----> Consumer -``` -- 1 (ClientAPI -> Sync): For tracking account data -- 2 (Roomserver -> Sync): For all data to send to clients -- 3 (EDUServer -> Sync): For typing/send-to-device data to send to clients -- 4 (Roomserver -> ClientAPI): For tracking memberships for profile updates. -- 5 (EDUServer -> FedSender): For sending EDUs over federation -- 6 (Roomserver -> FedSender): For sending PDUs over federation, for tracking joined hosts. diff --git a/docs/WIRING.md b/docs/WIRING.md deleted file mode 100644 index 8ec5b0432..000000000 --- a/docs/WIRING.md +++ /dev/null @@ -1,229 +0,0 @@ -# Wiring - -The diagram is incomplete. The following things aren't shown on the diagram: - -* Device Messages -* User Profiles -* Notification Counts -* Sending federation. -* Querying federation. -* Other things that aren't shown on the diagram. - -Diagram: - - - W -> Writer - S -> Server/Store/Service/Something/Stuff - R -> Reader - - +---+ +---+ +---+ - +----------| W | +----------| S | +--------| R | - | +---+ | Receipts +---+ | Client +---+ - | Federation |>=========================================>| Server |>=====================>| Sync | - | Receiver | | | | | - | | +---+ | | | | - | | +--------| W | | | | | - | | | Client +---+ | | | | - | | | Receipt |>=====>| | | | - | | | Updater | | | | | - | | +----------+ | | | | - | | | | | | - | | +---+ +---+ | | +---+ | | - | | +------------| W | +------| S | | | +--------| R | | | - | | | Federation +---+ | Room +---+ | | | Client +---+ | | - | | | Backfill |>=====>| Server |>=====>| |>=====>| Push | | | - | | +--------------+ | | +------------+ | | | | - | | | | | | | | - | | | |>==========================>| | | | - | | | | +----------+ | | - | | | | +---+ | | - | | | | +-------------| R | | | - | | | |>=====>| Application +---+ | | - | | | | | Services | | | - | | | | +--------------+ | | - | | | | +---+ | | - | | | | +--------| R | | | - | | | | | Client +---+ | | - | |>========================>| |>==========================>| Search | | | - | | | | | | | | - | | | | +----------+ | | - | | | | | | - | | | |>==========================================>| | - | | | | | | - | | +---+ | | +---+ | | - | | +--------| W | | | +----------| S | | | - | | | Client +---+ | | | Presence +---+ | | - | | | API |>=====>| |>=====>| Server |>=====================>| | - | | | /send | +--------+ | | | | - | | | | | | | | - | | | |>======================>| |<=====================<| | - | | +----------+ | | | | - | | | | | | - | | +---+ | | | | - | | +--------| W | | | | | - | | | Client +---+ | | | | - | | | Presence |>=====>| | | | - | | | Setter | | | | | - | | +----------+ | | | | - | | | | | | - | | | | | | - | |>=========================================>| | | | - | | +------------+ | | - | | | | - | | +---+ | | - | | +----------| S | | | - | | | EDU +---+ | | - | |>=========================================>| Server |>=====================>| | - +------------+ | | +----------+ - +---+ | | - +--------| W | | | - | Client +---+ | | - | Typing |>=====>| | - | Setter | | | - +----------+ +------------+ - - -# Component Descriptions - -Many of the components are logical rather than physical. For example it is -possible that all of the client API writers will end up being glued together -and always deployed as a single unit. - -Outbound federation requests will probably need to be funnelled through a -choke-point to implement ratelimiting and backoff correctly. - -## Federation Send - - * Handles `/federation/v1/send/` requests. - * Fetches missing ``prev_events`` from the remote server if needed. - * Fetches missing room state from the remote server if needed. - * Checks signatures on remote events, downloading keys if needed. - * Queries information needed to process events from the Room Server. - * Writes room events to logs. - * Writes presence updates to logs. - * Writes receipt updates to logs. - * Writes typing updates to logs. - * Writes other updates to logs. - -## Client API /send - - * Handles puts to `/client/v1/rooms/` that create room events. - * Queries information needed to process events from the Room Server. - * Talks to remote servers if needed for joins and invites. - * Writes room event pdus. - * Writes presence updates to logs. - -## Client Presence Setter - - * Handles puts to the [client API presence paths](https://matrix.org/docs/spec/client_server/unstable.html#id41). - * Writes presence updates to logs. - -## Client Typing Setter - - * Handles puts to the [client API typing paths](https://matrix.org/docs/spec/client_server/unstable.html#id32). - * Writes typing updates to logs. - -## Client Receipt Updater - - * Handles puts to the [client API receipt paths](https://matrix.org/docs/spec/client_server/unstable.html#id36). - * Writes receipt updates to logs. - -## Federation Backfill - - * Backfills events from other servers - * Writes the resulting room events to logs. - * Is a different component from the room server itself cause it'll - be easier if the room server component isn't making outbound HTTP requests - to remote servers - -## Room Server - - * Reads new and backfilled room events from the logs written by FS, FB and CRS. - * Tracks the current state of the room and the state at each event. - * Probably does auth checks on the incoming events. - * Handles state resolution as part of working out the current state and the - state at each event. - * Writes updates to the current state and new events to logs. - * Shards by room ID. - -## Receipt Server - - * Reads new updates to receipts from the logs written by the FS and CRU. - * Somehow learns enough information from the room server to workout how the - current receipt markers move with each update. - * Writes the new marker positions to logs - * Shards by room ID? - * It may be impossible to implement without folding it into the Room Server - forever coupling the components together. - -## EDU Server - - * Reads new updates to typing from the logs written by the FS and CTS. - * Updates the current list of people typing in a room. - * Writes the current list of people typing in a room to the logs. - * Shards by room ID? - -## Presence Server - - * Reads the current state of the rooms from the logs to track the intersection - of room membership between users. - * Reads updates to presence from the logs written by the FS and the CPS. - * Reads when clients sync from the logs from the Client Sync. - * Tracks any timers for users. - * Writes the changes to presence state to the logs. - * Shards by user ID somehow? - -## Client Sync - - * Handle /client/v2/sync requests. - * Reads new events and the current state of the rooms from logs written by the Room Server. - * Reads new receipts positions from the logs written by the Receipts Server. - * Reads changes to presence from the logs written by the Presence Server. - * Reads changes to typing from the logs written by the EDU Server. - * Writes when a client starts and stops syncing to the logs. - -## Client Search - - * Handle whatever the client API path for event search is? - * Reads new events and the current state of the rooms from logs writeen by the Room Server. - * Maintains a full text search index of somekind. - -## Client Push - - * Pushes unread messages to remote push servers. - * Reads new events and the current state of the rooms from logs writeen by the Room Server. - * Reads the position of the read marker from the Receipts Server. - * Makes outbound HTTP hits to the push server for the client device. - -## Application Service - - * Receives events from the Room Server. - * Filters events and sends them to each registered application service. - * Runs a separate goroutine for each application service. - -# Internal Component API - -Some dendrite components use internal APIs to communicate information back -and forth between each other. There are two implementations of each API, one -that uses HTTP requests and one that does not. The HTTP implementation is -used in multi-process mode, so processes on separate computers may still -communicate, whereas in single-process or Monolith mode, the direct -implementation is used. HTTP is preferred here to kafka streams as it allows -for request responses. - -Running `dendrite-monolith-server` will set up direct connections between -components, whereas running each individual component (which are only run in -multi-process mode) will set up HTTP-based connections. - -The functions that make HTTP requests to internal APIs of a component are -located in `//api/.go`, named according to what -functionality they cover. Each of these requests are handled in `///.go`. - -As an example, the `appservices` component allows other Dendrite components -to query external application services via its internal API. A component -would call the desired function in `/appservices/api/query.go`. In -multi-process mode, this would send an internal HTTP request, which would -be handled by a function in `/appservices/query/query.go`. In single-process -mode, no internal HTTP request occurs, instead functions are simply called -directly, thus requiring no changes on the calling component's end. diff --git a/docs/_config.yml b/docs/_config.yml new file mode 100644 index 000000000..ed93fd796 --- /dev/null +++ b/docs/_config.yml @@ -0,0 +1,19 @@ +title: Dendrite +description: >- + Second-generation Matrix homeserver written in Go! +baseurl: "/dendrite" # the subpath of your site, e.g. /blog +url: "" +twitter_username: matrixdotorg +github_username: matrix-org +remote_theme: just-the-docs/just-the-docs +plugins: + - jekyll-feed +aux_links: + "GitHub": + - "//github.com/matrix-org/dendrite" +aux_links_new_tab: true +sass: + sass_dir: _sass + style: compressed +exclude: + - INSTALL.md diff --git a/docs/_sass/custom/custom.scss b/docs/_sass/custom/custom.scss new file mode 100644 index 000000000..8a5ed3d8d --- /dev/null +++ b/docs/_sass/custom/custom.scss @@ -0,0 +1,3 @@ +footer.site-footer { + opacity: 10%; +} \ No newline at end of file diff --git a/docs/administration.md b/docs/administration.md new file mode 100644 index 000000000..08ad7803e --- /dev/null +++ b/docs/administration.md @@ -0,0 +1,10 @@ +--- +title: Administration +has_children: yes +nav_order: 4 +permalink: /administration +--- + +# Administration + +This section contains documentation on managing your existing Dendrite deployment. diff --git a/docs/administration/1_createusers.md b/docs/administration/1_createusers.md new file mode 100644 index 000000000..f40b7f576 --- /dev/null +++ b/docs/administration/1_createusers.md @@ -0,0 +1,53 @@ +--- +title: Creating user accounts +parent: Administration +permalink: /administration/createusers +nav_order: 1 +--- + +# Creating user accounts + +User accounts can be created on a Dendrite instance in a number of ways. + +## From the command line + +The `create-account` tool is built in the `bin` folder when building Dendrite with +the `build.sh` script. + +It uses the `dendrite.yaml` configuration file to connect to the Dendrite user database +and create the account entries directly. It can therefore be used even if Dendrite is not +running yet, as long as the database is up. + +An example of using `create-account` to create a **normal account**: + +```bash +./bin/create-account -config /path/to/dendrite.yaml -username USERNAME +``` + +You will be prompted to enter a new password for the new account. + +To create a new **admin account**, add the `-admin` flag: + +```bash +./bin/create-account -config /path/to/dendrite.yaml -username USERNAME -admin +``` + +## Using shared secret registration + +Dendrite supports the Synapse-compatible shared secret registration endpoint. + +To enable shared secret registration, you must first enable it in the `dendrite.yaml` +configuration file by specifying a shared secret. In the `client_api` section of the config, +enter a new secret into the `registration_shared_secret` field: + +```yaml +client_api: + # ... + registration_shared_secret: "" +``` + +You can then use the `/_synapse/admin/v1/register` endpoint as per the +[Synapse documentation](https://matrix-org.github.io/synapse/latest/admin_api/register_api.html). + +Shared secret registration is only enabled once a secret is configured. To disable shared +secret registration again, remove the secret from the configuration file. diff --git a/docs/administration/2_registration.md b/docs/administration/2_registration.md new file mode 100644 index 000000000..66949f2ca --- /dev/null +++ b/docs/administration/2_registration.md @@ -0,0 +1,53 @@ +--- +title: Enabling registration +parent: Administration +permalink: /administration/registration +nav_order: 2 +--- + +# Enabling registration + +Enabling registration allows users to register their own user accounts on your +Dendrite server using their Matrix client. They will be able to choose their own +username and password and log in. + +Registration is controlled by the `registration_disabled` field in the `client_api` +section of the configuration. By default, `registration_disabled` is set to `true`, +disabling registration. If you want to enable registration, you should change this +setting to `false`. + +Currently Dendrite supports secondary verification using [reCAPTCHA](https://www.google.com/recaptcha/about/). +Other methods will be supported in the future. + +## reCAPTCHA verification + +Dendrite supports reCAPTCHA as a secondary verification method. If you want to enable +registration, it is **highly recommended** to configure reCAPTCHA. This will make it +much more difficult for automated spam systems from registering accounts on your +homeserver automatically. + +You will need an API key from the [reCAPTCHA Admin Panel](https://www.google.com/recaptcha/admin). +Then configure the relevant details in the `client_api` section of the configuration: + +```yaml +client_api: + # ... + registration_disabled: false + recaptcha_public_key: "PUBLIC_KEY_HERE" + recaptcha_private_key: "PRIVATE_KEY_HERE" + enable_registration_captcha: true + captcha_bypass_secret: "" + recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify" +``` + +## Open registration + +Dendrite does support open registration — that is, allowing users to create their own +user accounts without any verification or secondary authentication. However, it +is **not recommended** to enable open registration, as this leaves your homeserver +vulnerable to abuse by spammers or attackers, who create large numbers of user +accounts on Matrix homeservers in order to send spam or abuse into the network. + +It isn't possible to enable open registration in Dendrite in a single step. If you +try to disable the `registration_disabled` option without any secondary verification +methods enabled (such as reCAPTCHA), Dendrite will log an error and fail to start. diff --git a/docs/administration/3_presence.md b/docs/administration/3_presence.md new file mode 100644 index 000000000..858025370 --- /dev/null +++ b/docs/administration/3_presence.md @@ -0,0 +1,39 @@ +--- +title: Enabling presence +parent: Administration +permalink: /administration/presence +nav_order: 3 +--- + +# Enabling presence + +Dendrite supports presence, which allows you to send your online/offline status +to other users, and to receive their statuses automatically. They will be displayed +by supported clients. + +Note that enabling presence **can negatively impact** the performance of your Dendrite +server — it will require more CPU time and will increase the "chattiness" of your server +over federation. It is disabled by default for this reason. + +Dendrite has two options for controlling presence: + +* **Enable inbound presence**: Dendrite will handle presence updates for remote users + and distribute them to local users on your homeserver; +* **Enable outbound presence**: Dendrite will generate presence notifications for your + local users and distribute them to remote users over the federation. + +This means that you can configure only one or other direction if you prefer, i.e. to +receive presence from other servers without revealing the presence of your own users. + +## Configuring presence + +Presence is controlled by the `presence` block in the `global` section of the +configuration file: + +```yaml +global: + # ... + presence: + enable_inbound: false + enable_outbound: false +``` diff --git a/docs/administration/4_adminapi.md b/docs/administration/4_adminapi.md new file mode 100644 index 000000000..e33482ec9 --- /dev/null +++ b/docs/administration/4_adminapi.md @@ -0,0 +1,25 @@ +--- +title: Supported admin APIs +parent: Administration +permalink: /administration/adminapi +--- + +# Supported admin APIs + +Dendrite supports, at present, a very small number of endpoints that allow +admin users to perform administrative functions. Please note that there is no +API stability guarantee on these endpoints at present — they may change shape +without warning. + +More endpoints will be added in the future. + +## `/_dendrite/admin/evacuateRoom/{roomID}` + +This endpoint will instruct Dendrite to part all local users from the given `roomID` +in the URL. It may take some time to complete. A JSON body will be returned containing +the user IDs of all affected users. + +## `/_synapse/admin/v1/register` + +Shared secret registration — please see the [user creation page](createusers) for +guidance on configuring and using this endpoint. diff --git a/docs/coverage.md b/docs/coverage.md new file mode 100644 index 000000000..7a3b7cb9e --- /dev/null +++ b/docs/coverage.md @@ -0,0 +1,84 @@ +--- +title: Coverage +parent: Development +permalink: /development/coverage +--- + +To generate a test coverage report for Sytest, a small patch needs to be applied to the Sytest repository to compile and use the instrumented binary: +```patch +diff --git a/lib/SyTest/Homeserver/Dendrite.pm b/lib/SyTest/Homeserver/Dendrite.pm +index 8f0e209c..ad057e52 100644 +--- a/lib/SyTest/Homeserver/Dendrite.pm ++++ b/lib/SyTest/Homeserver/Dendrite.pm +@@ -337,7 +337,7 @@ sub _start_monolith + + $output->diag( "Starting monolith server" ); + my @command = ( +- $self->{bindir} . '/dendrite-monolith-server', ++ $self->{bindir} . '/dendrite-monolith-server', '--test.coverprofile=' . $self->{hs_dir} . '/integrationcover.log', "DEVEL", + '--config', $self->{paths}{config}, + '--http-bind-address', $self->{bind_host} . ':' . $self->unsecure_port, + '--https-bind-address', $self->{bind_host} . ':' . $self->secure_port, +diff --git a/scripts/dendrite_sytest.sh b/scripts/dendrite_sytest.sh +index f009332b..7ea79869 100755 +--- a/scripts/dendrite_sytest.sh ++++ b/scripts/dendrite_sytest.sh +@@ -34,7 +34,8 @@ export GOBIN=/tmp/bin + echo >&2 "--- Building dendrite from source" + cd /src + mkdir -p $GOBIN +-go install -v ./cmd/dendrite-monolith-server ++# go install -v ./cmd/dendrite-monolith-server ++go test -c -cover -covermode=atomic -o $GOBIN/dendrite-monolith-server -coverpkg "github.com/matrix-org/..." ./cmd/dendrite-monolith-server + go install -v ./cmd/generate-keys + cd - + ``` + + Then run Sytest. This will generate a new file `integrationcover.log` in each server's directory e.g `server-0/integrationcover.log`. To parse it, + ensure your working directory is under the Dendrite repository then run: + ```bash + go tool cover -func=/path/to/server-0/integrationcover.log + ``` + which will produce an output like: + ``` + ... + github.com/matrix-org/util/json.go:83: NewJSONRequestHandler 100.0% +github.com/matrix-org/util/json.go:90: Protect 57.1% +github.com/matrix-org/util/json.go:110: RequestWithLogging 100.0% +github.com/matrix-org/util/json.go:132: MakeJSONAPI 70.0% +github.com/matrix-org/util/json.go:151: respond 61.5% +github.com/matrix-org/util/json.go:180: WithCORSOptions 0.0% +github.com/matrix-org/util/json.go:191: SetCORSHeaders 100.0% +github.com/matrix-org/util/json.go:202: RandomString 100.0% +github.com/matrix-org/util/json.go:210: init 100.0% +github.com/matrix-org/util/unique.go:13: Unique 91.7% +github.com/matrix-org/util/unique.go:48: SortAndUnique 100.0% +github.com/matrix-org/util/unique.go:55: UniqueStrings 100.0% +total: (statements) 53.7% +``` +The total coverage for this run is the last line at the bottom. However, this value is misleading because Dendrite can run in many different configurations, +which will never be tested in a single test run (e.g sqlite or postgres, monolith or polylith). To get a more accurate value, additional processing is required +to remove packages which will never be tested and extension MSCs: +```bash +# These commands are all similar but change which package paths are _removed_ from the output. + +# For Postgres (monolith) +go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'inthttp|sqlite|setup/mscs|api_trace' > coverage.txt + +# For Postgres (polylith) +go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'sqlite|setup/mscs|api_trace' > coverage.txt + +# For SQLite (monolith) +go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'inthttp|postgres|setup/mscs|api_trace' > coverage.txt + +# For SQLite (polylith) +go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'postgres|setup/mscs|api_trace' > coverage.txt +``` + +A total value can then be calculated using: +```bash +cat coverage.txt | awk -F '\t+' '{x = x + $3} END {print x/NR}' +``` + + +We currently do not have a way to combine Sytest/Complement/Unit Tests into a single coverage report. \ No newline at end of file diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 000000000..cf296fb53 --- /dev/null +++ b/docs/development.md @@ -0,0 +1,10 @@ +--- +title: Development +has_children: true +permalink: /development +--- + +# Development + +This section contains documentation that may be useful when helping to develop +Dendrite. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 000000000..64836152c --- /dev/null +++ b/docs/index.md @@ -0,0 +1,24 @@ +--- +layout: home +nav_exclude: true +--- + +# Dendrite + +Dendrite is a second-generation Matrix homeserver written in Go! Following the microservice +architecture model, Dendrite is designed to be efficient, reliable and scalable. Despite being beta, +many Matrix features are already supported. + +This site aims to include relevant documentation to help you to get started with and +run Dendrite. Check out the following sections: + +* **[Installation](installation.md)** for building and deploying your own Dendrite homeserver +* **[Administration](administration.md)** for managing an existing Dendrite deployment +* **[Development](development.md)** for developing against Dendrite + +You can also join us in our Matrix rooms dedicated to Dendrite, but please check first that +your question hasn't already been [answered in the FAQ](FAQ.md): + +* **[#dendrite:matrix.org](https://matrix.to/#/#dendrite:matrix.org)** for general project discussion and support +* **[#dendrite-dev:matrix.org](https://matrix.to/#/#dendrite-dev:matrix.org)** for chat on Dendrite development specifically +* **[#dendrite-alerts:matrix.org](https://matrix.to/#/#dendrite-alerts:matrix.org)** for release notifications and other important announcements diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 000000000..c38a6dbb2 --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,10 @@ +--- +title: Installation +has_children: true +nav_order: 2 +permalink: /installation +--- + +# Installation + +This section contains documentation on installing a new Dendrite deployment. diff --git a/docs/installation/1_planning.md b/docs/installation/1_planning.md new file mode 100644 index 000000000..89cc5b4a6 --- /dev/null +++ b/docs/installation/1_planning.md @@ -0,0 +1,110 @@ +--- +title: Planning your installation +parent: Installation +nav_order: 1 +permalink: /installation/planning +--- + +# Planning your installation + +## Modes + +Dendrite can be run in one of two configurations: + +* **Monolith mode**: All components run in the same process. In this mode, + it is possible to run an in-process NATS Server instead of running a standalone deployment. + This will usually be the preferred model for low-to-mid volume deployments, providing the best + balance between performance and resource usage. + +* **Polylith mode**: A cluster of individual components running in their own processes, dealing + with different aspects of the Matrix protocol. Components communicate with each other using + internal HTTP APIs and NATS Server. This will almost certainly be the preferred model for very + large deployments but scalability comes with a cost. API calls are expensive and therefore a + polylith deployment may end up using disproportionately more resources for a smaller number of + users compared to a monolith deployment. + +At present, we **recommend monolith mode deployments** in all cases. + +## Databases + +Dendrite can run with either a PostgreSQL or a SQLite backend. There are considerable tradeoffs +to consider: + +* **PostgreSQL**: Needs to run separately to Dendrite, needs to be installed and configured separately + and and will use more resources over all, but will be **considerably faster** than SQLite. PostgreSQL + has much better write concurrency which will allow Dendrite to process more tasks in parallel. This + will be necessary for federated deployments to perform adequately. + +* **SQLite**: Built into Dendrite, therefore no separate database engine is necessary and is quite + a bit easier to set up, but will be much slower than PostgreSQL in most cases. SQLite only allows a + single writer on a database at a given time, which will significantly restrict Dendrite's ability + to process multiple tasks in parallel. + +At this time, we **recommend the PostgreSQL database engine** for all production deployments. + +## Requirements + +Dendrite will run on Linux, macOS and Windows Server. It should also run fine on variants +of BSD such as FreeBSD and OpenBSD. We have not tested Dendrite on AIX, Solaris, Plan 9 or z/OS — +your mileage may vary with these platforms. + +It is difficult to state explicitly the amount of CPU, RAM or disk space that a Dendrite +installation will need, as this varies considerably based on a number of factors. In particular: + +* The number of users using the server; +* The number of rooms that the server is joined to — federated rooms in particular will typically + use more resources than rooms with only local users; +* The complexity of rooms that the server is joined to — rooms with more members coming and + going will typically be of a much higher complexity. + +Some tasks are more expensive than others, such as joining rooms over federation, running state +resolution or sending messages into very large federated rooms with lots of remote users. Therefore +you should plan accordingly and ensure that you have enough resources available to endure spikes +in CPU or RAM usage, as these may be considerably higher than the idle resource usage. + +At an absolute minimum, Dendrite will expect 1GB RAM. For a comfortable day-to-day deployment +which can participate in federated rooms for a number of local users, be prepared to assign 2-4 +CPU cores and 8GB RAM — more if your user count increases. + +If you are running PostgreSQL on the same machine, allow extra headroom for this too, as the +database engine will also have CPU and RAM requirements of its own. Running too many heavy +services on the same machine may result in resource starvation and processes may end up being +killed by the operating system if they try to use too much memory. + +## Dependencies + +In order to install Dendrite, you will need to satisfy the following dependencies. + +### Go + +At this time, Dendrite supports being built with Go 1.16 or later. We do not support building +Dendrite with older versions of Go than this. If you are installing Go using a package manager, +you should check (by running `go version`) that you are using a suitable version before you start. + +### PostgreSQL + +If using the PostgreSQL database engine, you should install PostgreSQL 12 or later. + +### NATS Server + +Monolith deployments come with a built-in [NATS Server](https://github.com/nats-io/nats-server) and +therefore do not need this to be manually installed. If you are planning a monolith installation, you +do not need to do anything. + +Polylith deployments, however, currently need a standalone NATS Server installation with JetStream +enabled. + +To do so, follow the [NATS Server installation instructions](https://docs.nats.io/running-a-nats-service/introduction/installation) and then [start your NATS deployment](https://docs.nats.io/running-a-nats-service/introduction/running). JetStream must be enabled, either by passing the `-js` flag to `nats-server`, +or by specifying the `store_dir` option in the the `jetstream` configuration. + +### Reverse proxy (polylith deployments) + +Polylith deployments require a reverse proxy, such as [NGINX](https://www.nginx.com) or +[HAProxy](http://www.haproxy.org). Configuring those is not covered in this documentation, +although a [sample configuration for NGINX](https://github.com/matrix-org/dendrite/blob/main/docs/nginx/polylith-sample.conf) +is provided. + +### Windows + +Finally, if you want to build Dendrite on Windows, you will need need `gcc` in the path. The best +way to achieve this is by installing and building Dendrite under [MinGW-w64](https://www.mingw-w64.org/). diff --git a/docs/installation/2_domainname.md b/docs/installation/2_domainname.md new file mode 100644 index 000000000..0d4300eca --- /dev/null +++ b/docs/installation/2_domainname.md @@ -0,0 +1,93 @@ +--- +title: Setting up the domain +parent: Installation +nav_order: 2 +permalink: /installation/domainname +--- + +# Setting up the domain + +Every Matrix server deployment requires a server name which uniquely identifies it. For +example, if you are using the server name `example.com`, then your users will have usernames +that take the format `@user:example.com`. + +For federation to work, the server name must be resolvable by other homeservers on the internet +— that is, the domain must be registered and properly configured with the relevant DNS records. + +Matrix servers discover each other when federating using the following methods: + +1. If a well-known delegation exists on `example.com`, use the path server from the + well-known file to connect to the remote homeserver; +2. If a DNS SRV delegation exists on `example.com`, use the hostname and port from the DNS SRV + record to connect to the remote homeserver; +3. If neither well-known or DNS SRV delegation are configured, attempt to connect to the remote + homeserver by connecting to `example.com` port TCP/8448 using HTTPS. + +## TLS certificates + +Matrix federation requires that valid TLS certificates are present on the domain. You must +obtain certificates from a publicly accepted Certificate Authority (CA). [LetsEncrypt](https://letsencrypt.org) +is an example of such a CA that can be used. Self-signed certificates are not suitable for +federation and will typically not be accepted by other homeservers. + +A common practice to help ease the management of certificates is to install a reverse proxy in +front of Dendrite which manages the TLS certificates and HTTPS proxying itself. Software such as +[NGINX](https://www.nginx.com) and [HAProxy](http://www.haproxy.org) can be used for the task. +Although the finer details of configuring these are not described here, you must reverse proxy +all `/_matrix` paths to your Dendrite server. + +It is possible for the reverse proxy to listen on the standard HTTPS port TCP/443 so long as your +domain delegation is configured to point to port TCP/443. + +## Delegation + +Delegation allows you to specify the server name and port that your Dendrite installation is +reachable at, or to host the Dendrite server at a different server name to the domain that +is being delegated. + +For example, if your Dendrite installation is actually reachable at `matrix.example.com` port 8448, +you will be able to delegate from `example.com` to `matrix.example.com` so that your users will have +`@user:example.com` user names instead of `@user:matrix.example.com` usernames. + +Delegation can be performed in one of two ways: + +* **Well-known delegation**: A well-known text file is served over HTTPS on the domain name + that you want to use, pointing to your server on `matrix.example.com` port 8448; +* **DNS SRV delegation**: A DNS SRV record is created on the domain name that you want to + use, pointing to your server on `matrix.example.com` port TCP/8448. + +If you are using a reverse proxy to forward `/_matrix` to Dendrite, your well-known or DNS SRV +delegation must refer to the hostname and port that the reverse proxy is listening on instead. + +Well-known delegation is typically easier to set up and usually preferred. However, you can use +either or both methods to delegate. If you configure both methods of delegation, it is important +that they both agree and refer to the same hostname and port. + +## Well-known delegation + +Using well-known delegation requires that you are running a web server at `example.com` which +is listening on the standard HTTPS port TCP/443. + +Assuming that your Dendrite installation is listening for HTTPS connections at `matrix.example.com` +on port 8448, the delegation file must be served at `https://example.com/.well-known/matrix/server` +and contain the following JSON document: + +```json +{ + "m.server": "https://matrix.example.com:8448" +} +``` + +## DNS SRV delegation + +Using DNS SRV delegation requires creating DNS SRV records on the `example.com` zone which +refer to your Dendrite installation. + +Assuming that your Dendrite installation is listening for HTTPS connections at `matrix.example.com` +port 8448, the DNS SRV record must have the following fields: + +* Name: `@` (or whichever term your DNS provider uses to signal the root) +* Service: `_matrix` +* Protocol: `_tcp` +* Port: `8448` +* Target: `matrix.example.com` diff --git a/docs/installation/3_database.md b/docs/installation/3_database.md new file mode 100644 index 000000000..f64fe9150 --- /dev/null +++ b/docs/installation/3_database.md @@ -0,0 +1,106 @@ +--- +title: Preparing database storage +parent: Installation +nav_order: 3 +permalink: /installation/database +--- + +# Preparing database storage + +Dendrite uses SQL databases to store data. Depending on the database engine being used, you +may need to perform some manual steps outlined below. + +## SQLite + +SQLite deployments do not require manual database creation. Simply configure the database +filenames in the Dendrite configuration file and start Dendrite. The databases will be created +and populated automatically. + +Note that Dendrite **cannot share a single SQLite database across multiple components**. Each +component must be configured with its own SQLite database filename. + +### Connection strings + +Connection strings for SQLite databases take the following forms: + +* Current working directory path: `file:dendrite_component.db` +* Full specified path: `file:///path/to/dendrite_component.db` + +## PostgreSQL + +Dendrite can automatically populate the database with the relevant tables and indexes, but +it is not capable of creating the databases themselves. You will need to create the databases +manually. + +At this point, you can choose to either use a single database for all Dendrite components, +or you can run each component with its own separate database: + +* **Single database**: You will need to create a single PostgreSQL database. Monolith deployments + can use a single global connection pool, which makes updating the configuration file much easier. + Only one database connection string to manage and likely simpler to back up the database. All + components will be sharing the same database resources (CPU, RAM, storage). + +* **Separate databases**: You will need to create a separate PostgreSQL database for each + component. You will need to configure each component that has storage in the Dendrite + configuration file with its own connection parameters. Allows running a different database engine + for each component on a different machine if needs be, each with their own CPU, RAM and storage — + almost certainly overkill unless you are running a very large Dendrite deployment. + +For either configuration, you will want to: + +1. Configure a role (with a username and password) which Dendrite can use to connect to the + database; +2. Create the database(s) themselves, ensuring that the Dendrite role has privileges over them. + As Dendrite will create and manage the database tables, indexes and sequences by itself, the + Dendrite role must have suitable privileges over the database. + +### Connection strings + +The format of connection strings for PostgreSQL databases is described in the [PostgreSQL libpq manual](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING). Note that Dendrite only +supports the "Connection URIs" format and **will not** work with the "Keyword/Value Connection +string" format. + +Example supported connection strings take the format: + +* `postgresql://user:pass@hostname/database?options=...` +* `postgres://user:pass@hostname/database?options=...` + +If you need to disable SSL/TLS on the database connection, you may need to append `?sslmode=disable` to the end of the connection string. + +### Role creation + +Create a role which Dendrite can use to connect to the database, choosing a new password when +prompted. On macOS, you may need to omit the `sudo -u postgres` from the below instructions. + +```bash +sudo -u postgres createuser -P dendrite +``` + +### Single database creation + +Create the database itself, using the `dendrite` role from above: + +```bash +sudo -u postgres createdb -O dendrite dendrite +``` + +### Multiple database creation + +The following eight components require a database. In this example they will be named: + +| Appservice API | `dendrite_appservice` | +| Federation API | `dendrite_federationapi` | +| Media API | `dendrite_mediaapi` | +| MSCs | `dendrite_mscs` | +| Roomserver | `dendrite_roomserver` | +| Sync API | `dendrite_syncapi` | +| Key server | `dendrite_keyserver` | +| User API | `dendrite_userapi` | + +... therefore you will need to create eight different databases: + +```bash +for i in appservice federationapi mediaapi mscs roomserver syncapi keyserver userapi; do + sudo -u postgres createdb -O dendrite dendrite_$i +done +``` diff --git a/docs/installation/4_signingkey.md b/docs/installation/4_signingkey.md new file mode 100644 index 000000000..07dc485ff --- /dev/null +++ b/docs/installation/4_signingkey.md @@ -0,0 +1,79 @@ +--- +title: Generating signing keys +parent: Installation +nav_order: 4 +permalink: /installation/signingkeys +--- + +# Generating signing keys + +All Matrix homeservers require a signing private key, which will be used to authenticate +federation requests and events. + +The `generate-keys` utility can be used to generate a private key. Assuming that Dendrite was +built using `build.sh`, you should find the `generate-keys` utility in the `bin` folder. + +To generate a Matrix signing private key: + +```bash +./bin/generate-keys --private-key matrix_key.pem +``` + +The generated `matrix_key.pem` file is your new signing key. + +## Important warning + +You must treat this key as if it is highly sensitive and private, so **never share it with +anyone**. No one should ever ask you for this key for any reason, even to debug a problematic +Dendrite server. + +Make sure take a safe backup of this key. You will likely need it if you want to reinstall +Dendrite, or any other Matrix homeserver, on the same domain name in the future. If you lose +this key, you may have trouble joining federated rooms. + +## Old signing keys + +If you already have old signing keys from a previous Matrix installation on the same domain +name, you can reuse those instead, as long as they have not been previously marked as expired — +a key that has been marked as expired in the past is unusable. + +Old keys from a previous Dendrite installation can be reused as-is without any further +configuration required. Simply use that key file in the Dendrite configuration. + +If you have server keys from an older Synapse instance, you can convert them to Dendrite's PEM +format and configure them as `old_private_keys` in your config. + +## Key format + +Dendrite stores the server signing key in the PEM format with the following structure. + +``` +-----BEGIN MATRIX PRIVATE KEY----- +Key-ID: ed25519: + + +-----END MATRIX PRIVATE KEY----- +``` + +## Converting Synapse keys + +If you have signing keys from a previous Synapse installation, you should ideally configure them +as `old_private_keys` in your Dendrite config file. Synapse stores signing keys in the following +format: + +``` +ed25519 +``` + +To convert this key to Dendrite's PEM format, use the following template. You must copy the Key ID +exactly without modifying it. **It is important to include the trailing equals sign on the Base64 +Encoded Key Data** if it is not already present in the original key, as the key data needs to be +padded to exactly 32 bytes: + +``` +-----BEGIN MATRIX PRIVATE KEY----- +Key-ID: ed25519: + += +-----END MATRIX PRIVATE KEY----- +``` diff --git a/docs/installation/5_install_monolith.md b/docs/installation/5_install_monolith.md new file mode 100644 index 000000000..7de066cf7 --- /dev/null +++ b/docs/installation/5_install_monolith.md @@ -0,0 +1,21 @@ +--- +title: Installing as a monolith +parent: Installation +has_toc: true +nav_order: 5 +permalink: /installation/install/monolith +--- + +# Installing as a monolith + +You can install the Dendrite monolith binary into `$GOPATH/bin` by using `go install`: + +```sh +go install ./cmd/dendrite-monolith-server +``` + +Alternatively, you can specify a custom path for the binary to be written to using `go build`: + +```sh +go build -o /usr/local/bin/ ./cmd/dendrite-monolith-server +``` diff --git a/docs/installation/6_install_polylith.md b/docs/installation/6_install_polylith.md new file mode 100644 index 000000000..375512f8f --- /dev/null +++ b/docs/installation/6_install_polylith.md @@ -0,0 +1,33 @@ +--- +title: Installing as a polylith +parent: Installation +has_toc: true +nav_order: 6 +permalink: /installation/install/polylith +--- + +# Installing as a polylith + +You can install the Dendrite polylith binary into `$GOPATH/bin` by using `go install`: + +```sh +go install ./cmd/dendrite-polylith-multi +``` + +Alternatively, you can specify a custom path for the binary to be written to using `go build`: + +```sh +go build -o /usr/local/bin/ ./cmd/dendrite-polylith-multi +``` + +The `dendrite-polylith-multi` binary is a "multi-personality" binary which can run as +any of the components depending on the supplied command line parameters. + +## Reverse proxy + +Polylith deployments require a reverse proxy in order to ensure that requests are +sent to the correct endpoint. You must ensure that a suitable reverse proxy is installed +and configured. + +A [sample configuration file](https://github.com/matrix-org/dendrite/blob/main/docs/nginx/polylith-sample.conf) +is provided for [NGINX](https://www.nginx.com). diff --git a/docs/installation/7_configuration.md b/docs/installation/7_configuration.md new file mode 100644 index 000000000..e676afbe6 --- /dev/null +++ b/docs/installation/7_configuration.md @@ -0,0 +1,147 @@ +--- +title: Populate the configuration +parent: Installation +nav_order: 7 +permalink: /installation/configuration +--- + +# Populate the configuration + +The configuration file is used to configure Dendrite. Sample configuration files are +present in the top level of the Dendrite repository: + +* [`dendrite-sample.monolith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.monolith.yaml) +* [`dendrite-sample.polylith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml) + +You will need to duplicate the sample, calling it `dendrite.yaml` for example, and then +tailor it to your installation. At a minimum, you will need to populate the following +sections: + +## Server name + +First of all, you will need to configure the server name of your Matrix homeserver. +This must match the domain name that you have selected whilst [configuring the domain +name delegation](domainname). + +In the `global` section, set the `server_name` to your delegated domain name: + +```yaml +global: + # ... + server_name: example.com +``` + +## Server signing keys + +Next, you should tell Dendrite where to find your [server signing keys](signingkeys). + +In the `global` section, set the `private_key` to the path to your server signing key: + +```yaml +global: + # ... + private_key: /path/to/matrix_key.pem +``` + +## JetStream configuration + +Monolith deployments can use the built-in NATS Server rather than running a standalone +server. If you are building a polylith deployment, or you want to use a standalone NATS +Server anyway, you can also configure that too. + +### Built-in NATS Server (monolith only) + +In the `global` section, under the `jetstream` key, ensure that no server addresses are +configured and set a `storage_path` to a persistent folder on the filesystem: + +```yaml +global: + # ... + jetstream: + in_memory: false + storage_path: /path/to/storage/folder + topic_prefix: Dendrite +``` + +### Standalone NATS Server (monolith and polylith) + +To use a standalone NATS Server instance, you will need to configure `addresses` field +to point to the port that your NATS Server is listening on: + +```yaml +global: + # ... + jetstream: + addresses: + - localhost:4222 + topic_prefix: Dendrite +``` + +You do not need to configure the `storage_path` when using a standalone NATS Server instance. +In the case that you are connecting to a multi-node NATS cluster, you can configure more than +one address in the `addresses` field. + +## Database connections + +Configuring database connections varies based on the [database configuration](database) +that you chose. + +### Global connection pool (monolith with a single PostgreSQL database only) + +If you are running a monolith deployment and want to use a single connection pool to a +single PostgreSQL database, then you must uncomment and configure the `database` section +within the `global` section: + +```yaml +global: + # ... + database: + connection_string: postgres://user:pass@hostname/database?sslmode=disable + max_open_conns: 100 + max_idle_conns: 5 + conn_max_lifetime: -1 +``` + +**You must then remove or comment out** the `database` sections from other areas of the +configuration file, e.g. under the `app_service_api`, `federation_api`, `key_server`, +`media_api`, `mscs`, `room_server`, `sync_api` and `user_api` blocks, otherwise these will +override the `global` database configuration. + +### Per-component connections (all other configurations) + +If you are building a polylith deployment, are using SQLite databases or separate PostgreSQL +databases per component, then you must instead configure the `database` sections under each +of the component blocks ,e.g. under the `app_service_api`, `federation_api`, `key_server`, +`media_api`, `mscs`, `room_server`, `sync_api` and `user_api` blocks. + +For example, with PostgreSQL: + +```yaml +room_server: + # ... + database: + connection_string: postgres://user:pass@hostname/dendrite_component?sslmode=disable + max_open_conns: 10 + max_idle_conns: 2 + conn_max_lifetime: -1 +``` + +... or with SQLite: + +```yaml +room_server: + # ... + database: + connection_string: file:roomserver.db + max_open_conns: 10 + max_idle_conns: 2 + conn_max_lifetime: -1 +``` + +## Other sections + +There are other options which may be useful so review them all. In particular, if you are +trying to federate from your Dendrite instance into public rooms then configuring the +`key_perspectives` (like `matrix.org` in the sample) can help to improve reliability +considerably by allowing your homeserver to fetch public keys for dead homeservers from +another living server. diff --git a/docs/installation/8_starting_monolith.md b/docs/installation/8_starting_monolith.md new file mode 100644 index 000000000..e0e7309d2 --- /dev/null +++ b/docs/installation/8_starting_monolith.md @@ -0,0 +1,41 @@ +--- +title: Starting the monolith +parent: Installation +has_toc: true +nav_order: 9 +permalink: /installation/start/monolith +--- + +# Starting the monolith + +Once you have completed all of the preparation and installation steps, +you can start your Dendrite monolith deployment by starting the `dendrite-monolith-server`: + +```bash +./dendrite-monolith-server -config /path/to/dendrite.yaml +``` + +If you want to change the addresses or ports that Dendrite listens on, you +can use the `-http-bind-address` and `-https-bind-address` command line arguments: + +```bash +./dendrite-monolith-server -config /path/to/dendrite.yaml \ + -http-bind-address 1.2.3.4:12345 \ + -https-bind-address 1.2.3.4:54321 +``` + +## Running under systemd + +A common deployment pattern is to run the monolith under systemd. For this, you +will need to create a service unit file. An example service unit file is available +in the [GitHub repository](https://github.com/matrix-org/dendrite/blob/main/docs/systemd/monolith-example.service). + +Once you have installed the service unit, you can notify systemd, enable and start +the service: + +```bash +systemctl daemon-reload +systemctl enable dendrite +systemctl start dendrite +journalctl -fu dendrite +``` diff --git a/docs/installation/9_starting_polylith.md b/docs/installation/9_starting_polylith.md new file mode 100644 index 000000000..228e52e85 --- /dev/null +++ b/docs/installation/9_starting_polylith.md @@ -0,0 +1,73 @@ +--- +title: Starting the polylith +parent: Installation +has_toc: true +nav_order: 9 +permalink: /installation/start/polylith +--- + +# Starting the polylith + +Once you have completed all of the preparation and installation steps, +you can start your Dendrite polylith deployment by starting the various components +using the `dendrite-polylith-multi` personalities. + +## Start the reverse proxy + +Ensure that your reverse proxy is started and is proxying the correct +endpoints to the correct components. Software such as [NGINX](https://www.nginx.com) or +[HAProxy](http://www.haproxy.org) can be used for this purpose. A [sample configuration +for NGINX](https://github.com/matrix-org/dendrite/blob/main/docs/nginx/polylith-sample.conf) +is provided. + +## Starting the components + +Each component must be started individually: + +### Client API + +```bash +./dendrite-polylith-multi -config /path/to/dendrite.yaml clientapi +``` + +### Sync API + +```bash +./dendrite-polylith-multi -config /path/to/dendrite.yaml syncapi +``` + +### Media API + +```bash +./dendrite-polylith-multi -config /path/to/dendrite.yaml mediaapi +``` + +### Federation API + +```bash +./dendrite-polylith-multi -config /path/to/dendrite.yaml federationapi +``` + +### Roomserver + +```bash +./dendrite-polylith-multi -config /path/to/dendrite.yaml roomserver +``` + +### Appservice API + +```bash +./dendrite-polylith-multi -config /path/to/dendrite.yaml appservice +``` + +### User API + +```bash +./dendrite-polylith-multi -config /path/to/dendrite.yaml userapi +``` + +### Key server + +```bash +./dendrite-polylith-multi -config /path/to/dendrite.yaml keyserver +``` diff --git a/docs/p2p.md b/docs/other/p2p.md similarity index 71% rename from docs/p2p.md rename to docs/other/p2p.md index 4e9a50524..9f104f025 100644 --- a/docs/p2p.md +++ b/docs/other/p2p.md @@ -1,27 +1,34 @@ -## Peer-to-peer Matrix +--- +title: P2P Matrix +nav_exclude: true +--- + +# P2P Matrix These are the instructions for setting up P2P Dendrite, current as of May 2020. There's both Go stuff and JS stuff to do to set this up. -### Dendrite +## Dendrite -#### Build +### Build - The `main` branch has a WASM-only binary for dendrite: `./cmd/dendritejs`. - Build it and copy assets to riot-web. + ``` -$ ./build-dendritejs.sh -$ cp bin/main.wasm ../riot-web/src/vector/dendrite.wasm +./build-dendritejs.sh +cp bin/main.wasm ../riot-web/src/vector/dendrite.wasm ``` -#### Test +### Test To check that the Dendrite side is working well as Wasm, you can run the Wasm-specific tests: + ``` -$ ./test-dendritejs.sh +./test-dendritejs.sh ``` -### Rendezvous +## Rendezvous This is how peers discover each other and communicate. @@ -29,18 +36,18 @@ By default, Dendrite uses the Matrix-hosted websocket star relay server at TODO This is currently hard-coded in `./cmd/dendritejs/main.go` - you can also use a local one if you run your own relay: ``` -$ npm install --global libp2p-websocket-star-rendezvous -$ rendezvous --port=9090 --host=127.0.0.1 +npm install --global libp2p-websocket-star-rendezvous +rendezvous --port=9090 --host=127.0.0.1 ``` Then use `/ip4/127.0.0.1/tcp/9090/ws/p2p-websocket-star/`. -### Riot-web +## Riot-web You need to check out this repo: ``` -$ git clone git@github.com:matrix-org/go-http-js-libp2p.git +git clone git@github.com:matrix-org/go-http-js-libp2p.git ``` Make sure to `yarn install` in the repo. Then: @@ -53,26 +60,30 @@ if (!global.fs && global.require) { global.fs = require("fs"); } ``` -- Add the diff at https://github.com/vector-im/riot-web/compare/matthew/p2p?expand=1 - ignore the `package.json` stuff. + +- Add the diff at - ignore the `package.json` stuff. - Add the following symlinks: they HAVE to be symlinks as the diff in `webpack.config.js` references specific paths. + ``` -$ cd node_modules -$ ln -s ../../go-http-js-libp2p +cd node_modules +ln -s ../../go-http-js-libp2p ``` NB: If you don't run the server with `yarn start` you need to make sure your server is sending the header `Service-Worker-Allowed: /`. TODO: Make a Docker image with all of this in it and a volume mount for `dendrite.wasm`. -### Running +## Running You need a Chrome and a Firefox running to test locally as service workers don't work in incognito tabs. + - For Chrome, use `chrome://serviceworker-internals/` to unregister/see logs. - For Firefox, use `about:debugging#/runtime/this-firefox` to unregister. Use the console window to see logs. Assuming you've `yarn start`ed Riot-Web, go to `http://localhost:8080` and register with `http://localhost:8080` as your HS URL. You can: - - join rooms by room alias e.g `/join #foo:bar`. - - invite specific users to a room. - - explore the published room list. All members of the room can re-publish aliases (unlike Synapse). + +- join rooms by room alias e.g `/join #foo:bar`. +- invite specific users to a room. +- explore the published room list. All members of the room can re-publish aliases (unlike Synapse). diff --git a/docs/other/peeking.md b/docs/other/peeking.md new file mode 100644 index 000000000..c4ae89811 --- /dev/null +++ b/docs/other/peeking.md @@ -0,0 +1,33 @@ +--- +nav_exclude: true +--- + +## Peeking + +Local peeking is implemented as per [MSC2753](https://github.com/matrix-org/matrix-doc/pull/2753). + +Implementationwise, this means: + +* Users call `/peek` and `/unpeek` on the clientapi from a given device. +* The clientapi delegates these via HTTP to the roomserver, which coordinates peeking in general for a given room +* The roomserver writes an NewPeek event into the kafka log headed to the syncserver +* The syncserver tracks the existence of the local peek in the syncapi_peeks table in its DB, and then starts waking up the peeking devices for the room in question, putting it in the `peek` section of the /sync response. + +Peeking over federation is implemented as per [MSC2444](https://github.com/matrix-org/matrix-doc/pull/2444). + +For requests to peek our rooms ("inbound peeks"): + +* Remote servers call `/peek` on federationapi + * The federationapi queries the federationsender to check if this is renewing an inbound peek or not. + * If not, it hits the PerformInboundPeek on the roomserver to ask it for the current state of the room. + * The roomserver atomically (in theory) adds a NewInboundPeek to its kafka stream to tell the federationserver to start peeking. + * The federationsender receives the event, tracks the inbound peek in the federationsender_inbound_peeks table, and starts sending events to the peeking server. + * The federationsender evicts stale inbound peeks which haven't been renewed. + +For peeking into other server's rooms ("outbound peeks"): + +* The `roomserver` will kick the `federationsender` much as it does for a federated `/join` in order to trigger a federated outbound `/peek` +* The `federationsender` tracks the existence of the outbound peek in in its federationsender_outbound_peeks table. +* The `federationsender` regularly renews the remote peek as long as there are still peeking devices syncing for it. +* TBD: how do we tell if there are no devices currently syncing for a given peeked room? The syncserver needs to tell the roomserver + somehow who then needs to warn the federationsender. diff --git a/docs/peeking.md b/docs/peeking.md deleted file mode 100644 index 60f359072..000000000 --- a/docs/peeking.md +++ /dev/null @@ -1,26 +0,0 @@ -## Peeking - -Local peeking is implemented as per [MSC2753](https://github.com/matrix-org/matrix-doc/pull/2753). - -Implementationwise, this means: - * Users call `/peek` and `/unpeek` on the clientapi from a given device. - * The clientapi delegates these via HTTP to the roomserver, which coordinates peeking in general for a given room - * The roomserver writes an NewPeek event into the kafka log headed to the syncserver - * The syncserver tracks the existence of the local peek in the syncapi_peeks table in its DB, and then starts waking up the peeking devices for the room in question, putting it in the `peek` section of the /sync response. - -Peeking over federation is implemented as per [MSC2444](https://github.com/matrix-org/matrix-doc/pull/2444). - -For requests to peek our rooms ("inbound peeks"): - * Remote servers call `/peek` on federationapi - * The federationapi queries the federationsender to check if this is renewing an inbound peek or not. - * If not, it hits the PerformInboundPeek on the roomserver to ask it for the current state of the room. - * The roomserver atomically (in theory) adds a NewInboundPeek to its kafka stream to tell the federationserver to start peeking. - * The federationsender receives the event, tracks the inbound peek in the federationsender_inbound_peeks table, and starts sending events to the peeking server. - * The federationsender evicts stale inbound peeks which haven't been renewed. - -For peeking into other server's rooms ("outbound peeks"): - * The `roomserver` will kick the `federationsender` much as it does for a federated `/join` in order to trigger a federated outbound `/peek` - * The `federationsender` tracks the existence of the outbound peek in in its federationsender_outbound_peeks table. - * The `federationsender` regularly renews the remote peek as long as there are still peeking devices syncing for it. - * TBD: how do we tell if there are no devices currently syncing for a given peeked room? The syncserver needs to tell the roomserver - somehow who then needs to warn the federationsender. \ No newline at end of file diff --git a/docs/serverkeyformat.md b/docs/serverkeyformat.md deleted file mode 100644 index feda93454..000000000 --- a/docs/serverkeyformat.md +++ /dev/null @@ -1,29 +0,0 @@ -# Server Key Format - -Dendrite stores the server signing key in the PEM format with the following structure. - -``` ------BEGIN MATRIX PRIVATE KEY----- -Key-ID: ed25519: - - ------END MATRIX PRIVATE KEY----- -``` - -## Converting Synapse Keys - -If you have signing keys from a previous synapse server, you should ideally configure them as `old_private_keys` in your Dendrite config file. Synapse stores signing keys in the following format. - -``` -ed25519 -``` - -To convert this key to Dendrite's PEM format, use the following template. **It is important to include the equals sign, as the key data needs to be padded to 32 bytes.** - -``` ------BEGIN MATRIX PRIVATE KEY----- -Key-ID: ed25519: - -= ------END MATRIX PRIVATE KEY----- -``` \ No newline at end of file diff --git a/docs/sytest.md b/docs/sytest.md index 0d42013ec..3cfb99e60 100644 --- a/docs/sytest.md +++ b/docs/sytest.md @@ -1,3 +1,9 @@ +--- +title: SyTest +parent: Development +permalink: /development/sytest +--- + # SyTest Dendrite uses [SyTest](https://github.com/matrix-org/sytest) for its @@ -43,6 +49,7 @@ source code. The test results TAP file and homeserver logging output will go to add any tests to `sytest-whitelist`. When debugging, the following Docker `run` options may also be useful: + * `-v /path/to/sytest/:/sytest/`: Use your local SyTest repository at `/path/to/sytest` instead of pulling from GitHub. This is useful when you want to speed things up or make modifications to SyTest. @@ -58,6 +65,7 @@ When debugging, the following Docker `run` options may also be useful: The docker command also supports a single positional argument for the test file to run, so you can run a single `.pl` file rather than the whole test suite. For example: + ``` docker run --rm --name sytest -v "/Users/kegan/github/sytest:/sytest" -v "/Users/kegan/github/dendrite:/src" -v "/Users/kegan/logs:/logs" @@ -118,7 +126,7 @@ POSTGRES=1 ./run-tests.pl -I Dendrite::Monolith -d ../dendrite/bin -W ../dendrit where `tee` lets you see the results while they're being piped to the file, and `POSTGRES=1` enables testing with PostgeSQL. If the `POSTGRES` environment variable is not set or is set to 0, SyTest will fall back to SQLite 3. For more -flags and options, see https://github.com/matrix-org/sytest#running. +flags and options, see . Once the tests are complete, run the helper script to see if you need to add any newly passing test names to `sytest-whitelist` in the project's root diff --git a/docs/tracing/jaeger.png b/docs/tracing/jaeger.png deleted file mode 100644 index 8b1e61feb..000000000 Binary files a/docs/tracing/jaeger.png and /dev/null differ diff --git a/docs/tracing/opentracing.md b/docs/tracing/opentracing.md index a2110bc0e..8528c2ba3 100644 --- a/docs/tracing/opentracing.md +++ b/docs/tracing/opentracing.md @@ -1,5 +1,11 @@ -Opentracing -=========== +--- +title: OpenTracing +has_children: true +parent: Development +permalink: /development/opentracing +--- + +# OpenTracing Dendrite extensively uses the [opentracing.io](http://opentracing.io) framework to trace work across the different logical components. @@ -23,7 +29,6 @@ This is useful to see where the time is being spent processing a request on a component. However, opentracing allows tracking of spans across components. This makes it possible to see exactly what work goes into processing a request: - ``` Component 1 |<─────────────────── HTTP ────────────────────>| |<──────────────── RPC ─────────────────>| @@ -39,7 +44,6 @@ deserialized span as the parent). A collection of spans that are related is called a trace. - Spans are passed through the code via contexts, rather than manually. It is therefore important that all spans that are created are immediately added to the current context. Thankfully the opentracing library gives helper functions for @@ -53,11 +57,11 @@ defer span.Finish() This will create a new span, adding any span already in `ctx` as a parent to the new span. - Adding Information ------------------ Opentracing allows adding information to a trace via three mechanisms: + - "tags" ─ A span can be tagged with a key/value pair. This is typically information that relates to the span, e.g. for spans created for incoming HTTP requests could include the request path and response codes as tags, spans for @@ -69,12 +73,10 @@ Opentracing allows adding information to a trace via three mechanisms: inspecting the traces, but can be used to add context to logs or tags in child spans. - See [specification.md](https://github.com/opentracing/specification/blob/master/specification.md) for some of the common tags and log fields used. - Span Relationships ------------------ @@ -86,7 +88,6 @@ A second relation type is `followsFrom`, where the parent has no dependence on the child span. This usually indicates some sort of fire and forget behaviour, e.g. adding a message to a pipeline or inserting into a kafka topic. - Jaeger ------ @@ -99,6 +100,7 @@ giving a UI for viewing and interacting with traces. To enable jaeger a `Tracer` object must be instansiated from the config (as well as having a jaeger server running somewhere, usually locally). A `Tracer` does several things: + - Decides which traces to save and send to the server. There are multiple schemes for doing this, with a simple example being to save a certain fraction of traces. diff --git a/docs/tracing/setup.md b/docs/tracing/setup.md index 2cab4d1ef..06f89bf85 100644 --- a/docs/tracing/setup.md +++ b/docs/tracing/setup.md @@ -1,14 +1,20 @@ -## OpenTracing Setup +--- +title: Setup +parent: OpenTracing +grand_parent: Development +permalink: /development/opentracing/setup +--- -![Trace when sending an event into a room](/docs/tracing/jaeger.png) +# OpenTracing Setup Dendrite uses [Jaeger](https://www.jaegertracing.io/) for tracing between microservices. Tracing shows the nesting of logical spans which provides visibility on how the microservices interact. This document explains how to set up Jaeger locally on a single machine. -### Set up the Jaeger backend +## Set up the Jaeger backend The [easiest way](https://www.jaegertracing.io/docs/1.18/getting-started/) is to use the all-in-one Docker image: + ``` $ docker run -d --name jaeger \ -e COLLECTOR_ZIPKIN_HTTP_PORT=9411 \ @@ -23,9 +29,10 @@ $ docker run -d --name jaeger \ jaegertracing/all-in-one:1.18 ``` -### Configuring Dendrite to talk to Jaeger +## Configuring Dendrite to talk to Jaeger Modify your config to look like: (this will send every single span to Jaeger which will be slow on large instances, but for local testing it's fine) + ``` tracing: enabled: true @@ -40,10 +47,11 @@ tracing: ``` then run the monolith server with `--api true` to use polylith components which do tracing spans: + ``` -$ ./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml --api true +./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml --api true ``` -### Checking traces +## Checking traces -Visit http://localhost:16686 to see traces under `DendriteMonolith`. +Visit to see traces under `DendriteMonolith`. diff --git a/federationapi/api/api.go b/federationapi/api/api.go index fc25194e0..53d4701f3 100644 --- a/federationapi/api/api.go +++ b/federationapi/api/api.go @@ -12,12 +12,16 @@ import ( // FederationInternalAPI is used to query information from the federation sender. type FederationInternalAPI interface { - FederationClient + gomatrixserverlib.FederatedStateClient + KeyserverFederationAPI gomatrixserverlib.KeyDatabase ClientFederationAPI RoomserverFederationAPI QueryServerKeys(ctx context.Context, request *QueryServerKeysRequest, response *QueryServerKeysResponse) error + LookupServerKeys(ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) ([]gomatrixserverlib.ServerKeys, error) + MSC2836EventRelationships(ctx context.Context, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error) + MSC2946Spaces(ctx context.Context, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res gomatrixserverlib.MSC2946SpacesResponse, err error) // Broadcasts an EDU to all servers in rooms we are joined to. Used in the yggdrasil demos. PerformBroadcastEDU( @@ -60,17 +64,43 @@ type RoomserverFederationAPI interface { LookupMissingEvents(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error) } -// FederationClient is a subset of gomatrixserverlib.FederationClient functions which the fedsender +// KeyserverFederationAPI is a subset of gomatrixserverlib.FederationClient functions which the keyserver // implements as proxy calls, with built-in backoff/retries/etc. Errors returned from functions in // this interface are of type FederationClientError -type FederationClient interface { - gomatrixserverlib.FederatedStateClient +type KeyserverFederationAPI interface { GetUserDevices(ctx context.Context, s gomatrixserverlib.ServerName, userID string) (res gomatrixserverlib.RespUserDevices, err error) ClaimKeys(ctx context.Context, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (res gomatrixserverlib.RespClaimKeys, err error) QueryKeys(ctx context.Context, s gomatrixserverlib.ServerName, keys map[string][]string) (res gomatrixserverlib.RespQueryKeys, err error) +} + +// an interface for gmsl.FederationClient - contains functions called by federationapi only. +type FederationClient interface { + gomatrixserverlib.KeyClient + SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error) + + // Perform operations + LookupRoomAlias(ctx context.Context, s gomatrixserverlib.ServerName, roomAlias string) (res gomatrixserverlib.RespDirectory, err error) + Peek(ctx context.Context, s gomatrixserverlib.ServerName, roomID, peekID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespPeek, err error) + MakeJoin(ctx context.Context, s gomatrixserverlib.ServerName, roomID, userID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMakeJoin, err error) + SendJoin(ctx context.Context, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res gomatrixserverlib.RespSendJoin, err error) + MakeLeave(ctx context.Context, s gomatrixserverlib.ServerName, roomID, userID string) (res gomatrixserverlib.RespMakeLeave, err error) + SendLeave(ctx context.Context, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (err error) + SendInviteV2(ctx context.Context, s gomatrixserverlib.ServerName, request gomatrixserverlib.InviteV2Request) (res gomatrixserverlib.RespInviteV2, err error) + + GetEvent(ctx context.Context, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error) + + GetEventAuth(ctx context.Context, s gomatrixserverlib.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res gomatrixserverlib.RespEventAuth, err error) + GetUserDevices(ctx context.Context, s gomatrixserverlib.ServerName, userID string) (gomatrixserverlib.RespUserDevices, error) + ClaimKeys(ctx context.Context, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (gomatrixserverlib.RespClaimKeys, error) + QueryKeys(ctx context.Context, s gomatrixserverlib.ServerName, keys map[string][]string) (gomatrixserverlib.RespQueryKeys, error) + Backfill(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, limit int, eventIDs []string) (res gomatrixserverlib.Transaction, err error) MSC2836EventRelationships(ctx context.Context, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error) MSC2946Spaces(ctx context.Context, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res gomatrixserverlib.MSC2946SpacesResponse, err error) - LookupServerKeys(ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) ([]gomatrixserverlib.ServerKeys, error) + + ExchangeThirdPartyInvite(ctx context.Context, s gomatrixserverlib.ServerName, builder gomatrixserverlib.EventBuilder) (err error) + LookupState(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, eventID string, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespState, err error) + LookupStateIDs(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, eventID string) (res gomatrixserverlib.RespStateIDs, err error) + LookupMissingEvents(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error) } // FederationClientError is returned from FederationClient methods in the event of a problem. diff --git a/federationapi/consumers/keychange.go b/federationapi/consumers/keychange.go index 0ece18e97..6d3cf0e46 100644 --- a/federationapi/consumers/keychange.go +++ b/federationapi/consumers/keychange.go @@ -39,7 +39,7 @@ type KeyChangeConsumer struct { db storage.Database queues *queue.OutgoingQueues serverName gomatrixserverlib.ServerName - rsAPI roomserverAPI.RoomserverInternalAPI + rsAPI roomserverAPI.FederationRoomserverAPI topic string } @@ -50,7 +50,7 @@ func NewKeyChangeConsumer( js nats.JetStreamContext, queues *queue.OutgoingQueues, store storage.Database, - rsAPI roomserverAPI.RoomserverInternalAPI, + rsAPI roomserverAPI.FederationRoomserverAPI, ) *KeyChangeConsumer { return &KeyChangeConsumer{ ctx: process.Context(), @@ -120,6 +120,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool { logger.WithError(err).Error("failed to calculate joined rooms for user") return true } + // send this key change to all servers who share rooms with this user. destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true) if err != nil { diff --git a/federationapi/consumers/roomserver.go b/federationapi/consumers/roomserver.go index 80317ee69..e50ec66ad 100644 --- a/federationapi/consumers/roomserver.go +++ b/federationapi/consumers/roomserver.go @@ -36,7 +36,7 @@ import ( type OutputRoomEventConsumer struct { ctx context.Context cfg *config.FederationAPI - rsAPI api.RoomserverInternalAPI + rsAPI api.FederationRoomserverAPI jetstream nats.JetStreamContext durable string db storage.Database @@ -51,7 +51,7 @@ func NewOutputRoomEventConsumer( js nats.JetStreamContext, queues *queue.OutgoingQueues, store storage.Database, - rsAPI api.RoomserverInternalAPI, + rsAPI api.FederationRoomserverAPI, ) *OutputRoomEventConsumer { return &OutputRoomEventConsumer{ ctx: process.Context(), @@ -89,15 +89,7 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msg *nats.Msg) switch output.Type { case api.OutputTypeNewRoomEvent: ev := output.NewRoomEvent.Event - - if output.NewRoomEvent.RewritesState { - if err := s.db.PurgeRoomState(s.ctx, ev.RoomID()); err != nil { - log.WithError(err).Errorf("roomserver output log: purge room state failure") - return false - } - } - - if err := s.processMessage(*output.NewRoomEvent); err != nil { + if err := s.processMessage(*output.NewRoomEvent, output.NewRoomEvent.RewritesState); err != nil { // panic rather than continue with an inconsistent database log.WithFields(log.Fields{ "event_id": ev.EventID(), @@ -145,7 +137,7 @@ func (s *OutputRoomEventConsumer) processInboundPeek(orp api.OutputNewInboundPee // processMessage updates the list of currently joined hosts in the room // and then sends the event to the hosts that were joined before the event. -func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) error { +func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rewritesState bool) error { addsStateEvents, missingEventIDs := ore.NeededStateEventIDs() // Ask the roomserver and add in the rest of the results into the set. @@ -164,7 +156,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) err addsStateEvents = append(addsStateEvents, eventsRes.Events...) } - addsJoinedHosts, err := joinedHostsFromEvents(gomatrixserverlib.UnwrapEventHeaders(addsStateEvents)) + addsJoinedHosts, err := JoinedHostsFromEvents(gomatrixserverlib.UnwrapEventHeaders(addsStateEvents)) if err != nil { return err } @@ -176,10 +168,9 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) err oldJoinedHosts, err := s.db.UpdateRoom( s.ctx, ore.Event.RoomID(), - ore.LastSentEventID, - ore.Event.EventID(), addsJoinedHosts, ore.RemovesStateEventIDs, + rewritesState, // if we're re-writing state, nuke all joined hosts before adding ) if err != nil { return err @@ -238,7 +229,7 @@ func (s *OutputRoomEventConsumer) joinedHostsAtEvent( return nil, err } - combinedAddsJoinedHosts, err := joinedHostsFromEvents(combinedAddsEvents) + combinedAddsJoinedHosts, err := JoinedHostsFromEvents(combinedAddsEvents) if err != nil { return nil, err } @@ -284,10 +275,10 @@ func (s *OutputRoomEventConsumer) joinedHostsAtEvent( return result, nil } -// joinedHostsFromEvents turns a list of state events into a list of joined hosts. +// JoinedHostsFromEvents turns a list of state events into a list of joined hosts. // This errors if one of the events was invalid. // It should be impossible for an invalid event to get this far in the pipeline. -func joinedHostsFromEvents(evs []*gomatrixserverlib.Event) ([]types.JoinedHost, error) { +func JoinedHostsFromEvents(evs []*gomatrixserverlib.Event) ([]types.JoinedHost, error) { var joinedHosts []types.JoinedHost for _, ev := range evs { if ev.Type() != "m.room.member" || ev.StateKey() == nil { diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index bec9ac777..ff159beea 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -93,8 +93,8 @@ func AddPublicRoutes( // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( base *base.BaseDendrite, - federation *gomatrixserverlib.FederationClient, - rsAPI roomserverAPI.RoomserverInternalAPI, + federation api.FederationClient, + rsAPI roomserverAPI.FederationRoomserverAPI, caches *caching.Caches, keyRing *gomatrixserverlib.KeyRing, resetBlacklist bool, diff --git a/federationapi/federationapi_keys_test.go b/federationapi/federationapi_keys_test.go index 4774c8820..31e9a4c73 100644 --- a/federationapi/federationapi_keys_test.go +++ b/federationapi/federationapi_keys_test.go @@ -102,7 +102,7 @@ func TestMain(m *testing.M) { ) // Finally, build the server key APIs. - sbase := base.NewBaseDendrite(cfg, "Monolith", base.NoCacheMetrics) + sbase := base.NewBaseDendrite(cfg, "Monolith", base.DisableMetrics) s.api = NewInternalAPI(sbase, s.fedclient, nil, s.cache, nil, true) } diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go index eedebc6cd..ae244c566 100644 --- a/federationapi/federationapi_test.go +++ b/federationapi/federationapi_test.go @@ -3,18 +3,250 @@ package federationapi_test import ( "context" "crypto/ed25519" + "encoding/json" + "fmt" "strings" "testing" + "time" "github.com/matrix-org/dendrite/federationapi" + "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/federationapi/internal" - "github.com/matrix-org/dendrite/internal/test" + keyapi "github.com/matrix-org/dendrite/keyserver/api" + rsapi "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" + "github.com/nats-io/nats.go" ) +type fedRoomserverAPI struct { + rsapi.FederationRoomserverAPI + inputRoomEvents func(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse) + queryRoomsForUser func(ctx context.Context, req *rsapi.QueryRoomsForUserRequest, res *rsapi.QueryRoomsForUserResponse) error +} + +// PerformJoin will call this function +func (f *fedRoomserverAPI) InputRoomEvents(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse) { + if f.inputRoomEvents == nil { + return + } + f.inputRoomEvents(ctx, req, res) +} + +// keychange consumer calls this +func (f *fedRoomserverAPI) QueryRoomsForUser(ctx context.Context, req *rsapi.QueryRoomsForUserRequest, res *rsapi.QueryRoomsForUserResponse) error { + if f.queryRoomsForUser == nil { + return nil + } + return f.queryRoomsForUser(ctx, req, res) +} + +// TODO: This struct isn't generic, only works for TestFederationAPIJoinThenKeyUpdate +type fedClient struct { + api.FederationClient + allowJoins []*test.Room + keys map[gomatrixserverlib.ServerName]struct { + key ed25519.PrivateKey + keyID gomatrixserverlib.KeyID + } + t *testing.T + sentTxn bool +} + +func (f *fedClient) GetServerKeys(ctx context.Context, matrixServer gomatrixserverlib.ServerName) (gomatrixserverlib.ServerKeys, error) { + fmt.Println("GetServerKeys:", matrixServer) + var keys gomatrixserverlib.ServerKeys + var keyID gomatrixserverlib.KeyID + var pkey ed25519.PrivateKey + for srv, data := range f.keys { + if srv == matrixServer { + pkey = data.key + keyID = data.keyID + break + } + } + if pkey == nil { + return keys, nil + } + + keys.ServerName = matrixServer + keys.ValidUntilTS = gomatrixserverlib.AsTimestamp(time.Now().Add(10 * time.Hour)) + publicKey := pkey.Public().(ed25519.PublicKey) + keys.VerifyKeys = map[gomatrixserverlib.KeyID]gomatrixserverlib.VerifyKey{ + keyID: { + Key: gomatrixserverlib.Base64Bytes(publicKey), + }, + } + toSign, err := json.Marshal(keys.ServerKeyFields) + if err != nil { + return keys, err + } + + keys.Raw, err = gomatrixserverlib.SignJSON( + string(matrixServer), keyID, pkey, toSign, + ) + if err != nil { + return keys, err + } + + return keys, nil +} + +func (f *fedClient) MakeJoin(ctx context.Context, s gomatrixserverlib.ServerName, roomID, userID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMakeJoin, err error) { + for _, r := range f.allowJoins { + if r.ID == roomID { + res.RoomVersion = r.Version + res.JoinEvent = gomatrixserverlib.EventBuilder{ + Sender: userID, + RoomID: roomID, + Type: "m.room.member", + StateKey: &userID, + Content: gomatrixserverlib.RawJSON([]byte(`{"membership":"join"}`)), + PrevEvents: r.ForwardExtremities(), + } + var needed gomatrixserverlib.StateNeeded + needed, err = gomatrixserverlib.StateNeededForEventBuilder(&res.JoinEvent) + if err != nil { + f.t.Errorf("StateNeededForEventBuilder: %v", err) + return + } + res.JoinEvent.AuthEvents = r.MustGetAuthEventRefsForEvent(f.t, needed) + return + } + } + return +} +func (f *fedClient) SendJoin(ctx context.Context, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res gomatrixserverlib.RespSendJoin, err error) { + for _, r := range f.allowJoins { + if r.ID == event.RoomID() { + r.InsertEvent(f.t, event.Headered(r.Version)) + f.t.Logf("Join event: %v", event.EventID()) + res.StateEvents = gomatrixserverlib.NewEventJSONsFromHeaderedEvents(r.CurrentState()) + res.AuthEvents = gomatrixserverlib.NewEventJSONsFromHeaderedEvents(r.Events()) + } + } + return +} + +func (f *fedClient) SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error) { + for _, edu := range t.EDUs { + if edu.Type == gomatrixserverlib.MDeviceListUpdate { + f.sentTxn = true + } + } + f.t.Logf("got /send") + return +} + +// Regression test to make sure that /send_join is updating the destination hosts synchronously and +// isn't relying on the roomserver. +func TestFederationAPIJoinThenKeyUpdate(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + testFederationAPIJoinThenKeyUpdate(t, dbType) + }) +} + +func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) { + base, close := testrig.CreateBaseDendrite(t, dbType) + base.Cfg.FederationAPI.PreferDirectFetch = true + defer close() + jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) + defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream) + + serverA := gomatrixserverlib.ServerName("server.a") + serverAKeyID := gomatrixserverlib.KeyID("ed25519:servera") + serverAPrivKey := test.PrivateKeyA + creator := test.NewUser(t, test.WithSigningServer(serverA, serverAKeyID, serverAPrivKey)) + + myServer := base.Cfg.Global.ServerName + myServerKeyID := base.Cfg.Global.KeyID + myServerPrivKey := base.Cfg.Global.PrivateKey + joiningUser := test.NewUser(t, test.WithSigningServer(myServer, myServerKeyID, myServerPrivKey)) + fmt.Printf("creator: %v joining user: %v\n", creator.ID, joiningUser.ID) + room := test.NewRoom(t, creator) + + rsapi := &fedRoomserverAPI{ + inputRoomEvents: func(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse) { + if req.Asynchronous { + t.Errorf("InputRoomEvents from PerformJoin MUST be synchronous") + } + }, + queryRoomsForUser: func(ctx context.Context, req *rsapi.QueryRoomsForUserRequest, res *rsapi.QueryRoomsForUserResponse) error { + if req.UserID == joiningUser.ID && req.WantMembership == "join" { + res.RoomIDs = []string{room.ID} + return nil + } + return fmt.Errorf("unexpected queryRoomsForUser: %+v", *req) + }, + } + fc := &fedClient{ + allowJoins: []*test.Room{room}, + t: t, + keys: map[gomatrixserverlib.ServerName]struct { + key ed25519.PrivateKey + keyID gomatrixserverlib.KeyID + }{ + serverA: { + key: serverAPrivKey, + keyID: serverAKeyID, + }, + myServer: { + key: myServerPrivKey, + keyID: myServerKeyID, + }, + }, + } + fsapi := federationapi.NewInternalAPI(base, fc, rsapi, base.Caches, nil, false) + + var resp api.PerformJoinResponse + fsapi.PerformJoin(context.Background(), &api.PerformJoinRequest{ + RoomID: room.ID, + UserID: joiningUser.ID, + ServerNames: []gomatrixserverlib.ServerName{serverA}, + }, &resp) + if resp.JoinedVia != serverA { + t.Errorf("PerformJoin: joined via %v want %v", resp.JoinedVia, serverA) + } + if resp.LastError != nil { + t.Fatalf("PerformJoin: returned error: %+v", *resp.LastError) + } + + // Inject a keyserver key change event and ensure we try to send it out. If we don't, then the + // federationapi is incorrectly waiting for an output room event to arrive to update the joined + // hosts table. + key := keyapi.DeviceMessage{ + Type: keyapi.TypeDeviceKeyUpdate, + DeviceKeys: &keyapi.DeviceKeys{ + UserID: joiningUser.ID, + DeviceID: "MY_DEVICE", + DisplayName: "BLARGLE", + KeyJSON: []byte(`{}`), + }, + } + b, err := json.Marshal(key) + if err != nil { + t.Fatalf("Failed to marshal device message: %s", err) + } + + msg := &nats.Msg{ + Subject: base.Cfg.Global.JetStream.Prefixed(jetstream.OutputKeyChangeEvent), + Header: nats.Header{}, + Data: b, + } + msg.Header.Set(jetstream.UserID, key.UserID) + + testrig.MustPublishMsgs(t, jsctx, msg) + time.Sleep(500 * time.Millisecond) + if !fc.sentTxn { + t.Fatalf("did not send device list update") + } +} + // Tests that event IDs with '/' in them (escaped as %2F) are correctly passed to the right handler and don't 404. // Relevant for v3 rooms and a cause of flakey sytests as the IDs are randomly generated. func TestRoomsV3URLEscapeDoNot404(t *testing.T) { @@ -86,7 +318,7 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) { } gerr, ok := err.(gomatrix.HTTPError) if !ok { - t.Errorf("failed to cast response error as gomatrix.HTTPError") + t.Errorf("failed to cast response error as gomatrix.HTTPError: %s", err) continue } t.Logf("Error: %+v", gerr) diff --git a/federationapi/internal/api.go b/federationapi/internal/api.go index 4e9fa8410..14056eafc 100644 --- a/federationapi/internal/api.go +++ b/federationapi/internal/api.go @@ -25,8 +25,8 @@ type FederationInternalAPI struct { db storage.Database cfg *config.FederationAPI statistics *statistics.Statistics - rsAPI roomserverAPI.RoomserverInternalAPI - federation *gomatrixserverlib.FederationClient + rsAPI roomserverAPI.FederationRoomserverAPI + federation api.FederationClient keyRing *gomatrixserverlib.KeyRing queues *queue.OutgoingQueues joins sync.Map // joins currently in progress @@ -34,8 +34,8 @@ type FederationInternalAPI struct { func NewFederationInternalAPI( db storage.Database, cfg *config.FederationAPI, - rsAPI roomserverAPI.RoomserverInternalAPI, - federation *gomatrixserverlib.FederationClient, + rsAPI roomserverAPI.FederationRoomserverAPI, + federation api.FederationClient, statistics *statistics.Statistics, caches *caching.Caches, queues *queue.OutgoingQueues, diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go index 577cb70e0..7ccd68ef0 100644 --- a/federationapi/internal/perform.go +++ b/federationapi/internal/perform.go @@ -8,6 +8,7 @@ import ( "time" "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/federationapi/consumers" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/gomatrix" @@ -235,6 +236,21 @@ func (r *FederationInternalAPI) performJoinUsingServer( return fmt.Errorf("respSendJoin.Check: %w", err) } + // We need to immediately update our list of joined hosts for this room now as we are technically + // joined. We must do this synchronously: we cannot rely on the roomserver output events as they + // will happen asyncly. If we don't update this table, you can end up with bad failure modes like + // joining a room, waiting for 200 OK then changing device keys and have those keys not be sent + // to other servers (this was a cause of a flakey sytest "Local device key changes get to remote servers") + // The events are trusted now as we performed auth checks above. + joinedHosts, err := consumers.JoinedHostsFromEvents(respState.StateEvents.TrustedEvents(respMakeJoin.RoomVersion, false)) + if err != nil { + return fmt.Errorf("JoinedHostsFromEvents: failed to get joined hosts: %s", err) + } + logrus.WithField("hosts", joinedHosts).WithField("room", roomID).Info("Joined federated room with hosts") + if _, err = r.db.UpdateRoom(context.Background(), roomID, joinedHosts, nil, true); err != nil { + return fmt.Errorf("UpdatedRoom: failed to update room with joined hosts: %s", err) + } + // If we successfully performed a send_join above then the other // server now thinks we're a part of the room. Send the newly // returned state to the roomserver to update our local view. @@ -650,7 +666,7 @@ func setDefaultRoomVersionFromJoinEvent(joinEvent gomatrixserverlib.EventBuilder // FederatedAuthProvider is an auth chain provider which fetches events from the server provided func federatedAuthProvider( - ctx context.Context, federation *gomatrixserverlib.FederationClient, + ctx context.Context, federation api.FederationClient, keyRing gomatrixserverlib.JSONVerifier, server gomatrixserverlib.ServerName, ) gomatrixserverlib.AuthChainProvider { // A list of events that we have retried, if they were not included in diff --git a/federationapi/queue/destinationqueue.go b/federationapi/queue/destinationqueue.go index 747940403..b6edec5da 100644 --- a/federationapi/queue/destinationqueue.go +++ b/federationapi/queue/destinationqueue.go @@ -21,6 +21,7 @@ import ( "sync" "time" + fedapi "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/federationapi/storage" "github.com/matrix-org/dendrite/federationapi/storage/shared" @@ -49,21 +50,21 @@ type destinationQueue struct { db storage.Database process *process.ProcessContext signing *SigningInfo - rsAPI api.RoomserverInternalAPI - client *gomatrixserverlib.FederationClient // federation client - origin gomatrixserverlib.ServerName // origin of requests - destination gomatrixserverlib.ServerName // destination of requests - running atomic.Bool // is the queue worker running? - backingOff atomic.Bool // true if we're backing off - overflowed atomic.Bool // the queues exceed maxPDUsInMemory/maxEDUsInMemory, so we should consult the database for more - statistics *statistics.ServerStatistics // statistics about this remote server - transactionIDMutex sync.Mutex // protects transactionID - transactionID gomatrixserverlib.TransactionID // last transaction ID if retrying, or "" if last txn was successful - notify chan struct{} // interrupts idle wait pending PDUs/EDUs - pendingPDUs []*queuedPDU // PDUs waiting to be sent - pendingEDUs []*queuedEDU // EDUs waiting to be sent - pendingMutex sync.RWMutex // protects pendingPDUs and pendingEDUs - interruptBackoff chan bool // interrupts backoff + rsAPI api.FederationRoomserverAPI + client fedapi.FederationClient // federation client + origin gomatrixserverlib.ServerName // origin of requests + destination gomatrixserverlib.ServerName // destination of requests + running atomic.Bool // is the queue worker running? + backingOff atomic.Bool // true if we're backing off + overflowed atomic.Bool // the queues exceed maxPDUsInMemory/maxEDUsInMemory, so we should consult the database for more + statistics *statistics.ServerStatistics // statistics about this remote server + transactionIDMutex sync.Mutex // protects transactionID + transactionID gomatrixserverlib.TransactionID // last transaction ID if retrying, or "" if last txn was successful + notify chan struct{} // interrupts idle wait pending PDUs/EDUs + pendingPDUs []*queuedPDU // PDUs waiting to be sent + pendingEDUs []*queuedEDU // EDUs waiting to be sent + pendingMutex sync.RWMutex // protects pendingPDUs and pendingEDUs + interruptBackoff chan bool // interrupts backoff } // Send event adds the event to the pending queue for the destination. diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go index d152886f5..4c25c4ce6 100644 --- a/federationapi/queue/queue.go +++ b/federationapi/queue/queue.go @@ -26,6 +26,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/tidwall/gjson" + fedapi "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/federationapi/storage" "github.com/matrix-org/dendrite/federationapi/storage/shared" @@ -39,9 +40,9 @@ type OutgoingQueues struct { db storage.Database process *process.ProcessContext disabled bool - rsAPI api.RoomserverInternalAPI + rsAPI api.FederationRoomserverAPI origin gomatrixserverlib.ServerName - client *gomatrixserverlib.FederationClient + client fedapi.FederationClient statistics *statistics.Statistics signing *SigningInfo queuesMutex sync.Mutex // protects the below @@ -85,8 +86,8 @@ func NewOutgoingQueues( process *process.ProcessContext, disabled bool, origin gomatrixserverlib.ServerName, - client *gomatrixserverlib.FederationClient, - rsAPI api.RoomserverInternalAPI, + client fedapi.FederationClient, + rsAPI api.FederationRoomserverAPI, statistics *statistics.Statistics, signing *SigningInfo, ) *OutgoingQueues { diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go index 707b7b019..316c61a14 100644 --- a/federationapi/routing/query.go +++ b/federationapi/routing/query.go @@ -30,7 +30,7 @@ import ( // RoomAliasToID converts the queried alias into a room ID and returns it func RoomAliasToID( httpReq *http.Request, - federation *gomatrixserverlib.FederationClient, + federation federationAPI.FederationClient, cfg *config.FederationAPI, rsAPI roomserverAPI.FederationRoomserverAPI, senderAPI federationAPI.FederationInternalAPI, diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 9f95ed07e..e25f9866e 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -54,7 +54,7 @@ func Setup( rsAPI roomserverAPI.FederationRoomserverAPI, fsAPI *fedInternal.FederationInternalAPI, keys gomatrixserverlib.JSONVerifier, - federation *gomatrixserverlib.FederationClient, + federation federationAPI.FederationClient, userAPI userapi.FederationUserAPI, keyAPI keyserverAPI.FederationKeyAPI, mscCfg *config.MSCs, diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 55a113675..c25dabce9 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -85,7 +85,7 @@ func Send( rsAPI api.FederationRoomserverAPI, keyAPI keyapi.FederationKeyAPI, keys gomatrixserverlib.JSONVerifier, - federation *gomatrixserverlib.FederationClient, + federation federationAPI.FederationClient, mu *internal.MutexByRoom, servers federationAPI.ServersInRoomProvider, producer *producers.SyncAPIProducer, diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index 011d4e342..a111580c7 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -8,8 +8,8 @@ import ( "time" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/internal/test" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/test" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index 16f245cee..ccde9168e 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" + federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -57,7 +58,7 @@ var ( func CreateInvitesFrom3PIDInvites( req *http.Request, rsAPI api.FederationRoomserverAPI, cfg *config.FederationAPI, - federation *gomatrixserverlib.FederationClient, + federation federationAPI.FederationClient, userAPI userapi.FederationUserAPI, ) util.JSONResponse { var body invites @@ -107,7 +108,7 @@ func ExchangeThirdPartyInvite( roomID string, rsAPI api.FederationRoomserverAPI, cfg *config.FederationAPI, - federation *gomatrixserverlib.FederationClient, + federation federationAPI.FederationClient, ) util.JSONResponse { var builder gomatrixserverlib.EventBuilder if err := json.Unmarshal(request.Content(), &builder); err != nil { @@ -165,7 +166,12 @@ func ExchangeThirdPartyInvite( // Ask the requesting server to sign the newly created event so we know it // acknowledged it - signedEvent, err := federation.SendInvite(httpReq.Context(), request.Origin(), event) + inviteReq, err := gomatrixserverlib.NewInviteV2Request(event.Headered(verRes.RoomVersion), nil) + if err != nil { + util.GetLogger(httpReq.Context()).WithError(err).Error("failed to make invite v2 request") + return jsonerror.InternalServerError() + } + signedEvent, err := federation.SendInviteV2(httpReq.Context(), request.Origin(), inviteReq) if err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("federation.SendInvite failed") return jsonerror.InternalServerError() @@ -205,7 +211,7 @@ func ExchangeThirdPartyInvite( func createInviteFrom3PIDInvite( ctx context.Context, rsAPI api.FederationRoomserverAPI, cfg *config.FederationAPI, - inv invite, federation *gomatrixserverlib.FederationClient, + inv invite, federation federationAPI.FederationClient, userAPI userapi.FederationUserAPI, ) (*gomatrixserverlib.Event, error) { verReq := api.QueryRoomVersionForRoomRequest{RoomID: inv.RoomID} @@ -335,7 +341,7 @@ func buildMembershipEvent( // them responded with an error. func sendToRemoteServer( ctx context.Context, inv invite, - federation *gomatrixserverlib.FederationClient, _ *config.FederationAPI, + federation federationAPI.FederationClient, _ *config.FederationAPI, builder gomatrixserverlib.EventBuilder, ) (err error) { remoteServers := make([]gomatrixserverlib.ServerName, 2) diff --git a/federationapi/storage/interface.go b/federationapi/storage/interface.go index e3038651b..29254948b 100644 --- a/federationapi/storage/interface.go +++ b/federationapi/storage/interface.go @@ -25,13 +25,12 @@ import ( type Database interface { gomatrixserverlib.KeyDatabase - UpdateRoom(ctx context.Context, roomID, oldEventID, newEventID string, addHosts []types.JoinedHost, removeHosts []string) (joinedHosts []types.JoinedHost, err error) + UpdateRoom(ctx context.Context, roomID string, addHosts []types.JoinedHost, removeHosts []string, purgeRoomFirst bool) (joinedHosts []types.JoinedHost, err error) GetJoinedHosts(ctx context.Context, roomID string) ([]types.JoinedHost, error) GetAllJoinedHosts(ctx context.Context) ([]gomatrixserverlib.ServerName, error) // GetJoinedHostsForRooms returns the complete set of servers in the rooms given. GetJoinedHostsForRooms(ctx context.Context, roomIDs []string, excludeSelf bool) ([]gomatrixserverlib.ServerName, error) - PurgeRoomState(ctx context.Context, roomID string) error StoreJSON(ctx context.Context, js string) (*shared.Receipt, error) diff --git a/federationapi/storage/shared/storage.go b/federationapi/storage/shared/storage.go index 160c7f6fa..a00d782f1 100644 --- a/federationapi/storage/shared/storage.go +++ b/federationapi/storage/shared/storage.go @@ -63,11 +63,21 @@ func (r *Receipt) String() string { // this isn't a duplicate message. func (d *Database) UpdateRoom( ctx context.Context, - roomID, oldEventID, newEventID string, + roomID string, addHosts []types.JoinedHost, removeHosts []string, + purgeRoomFirst bool, ) (joinedHosts []types.JoinedHost, err error) { err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + if purgeRoomFirst { + // If the event is a create event then we'll delete all of the existing + // data for the room. The only reason that a create event would be replayed + // to us in this way is if we're about to receive the entire room state. + if err = d.FederationJoinedHosts.DeleteJoinedHostsForRoom(ctx, txn, roomID); err != nil { + return fmt.Errorf("d.FederationJoinedHosts.DeleteJoinedHosts: %w", err) + } + } + joinedHosts, err = d.FederationJoinedHosts.SelectJoinedHostsWithTx(ctx, txn, roomID) if err != nil { return err @@ -138,20 +148,6 @@ func (d *Database) StoreJSON( }, nil } -func (d *Database) PurgeRoomState( - ctx context.Context, roomID string, -) error { - return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - // If the event is a create event then we'll delete all of the existing - // data for the room. The only reason that a create event would be replayed - // to us in this way is if we're about to receive the entire room state. - if err := d.FederationJoinedHosts.DeleteJoinedHostsForRoom(ctx, txn, roomID); err != nil { - return fmt.Errorf("d.FederationJoinedHosts.DeleteJoinedHosts: %w", err) - } - return nil - }) -} - func (d *Database) AddServerToBlacklist(serverName gomatrixserverlib.ServerName) error { return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { return d.FederationBlacklist.InsertBlacklist(context.TODO(), txn, serverName) diff --git a/go.mod b/go.mod index d14ced5b7..ecdeb77fa 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/matrix-org/dendrite -replace github.com/nats-io/nats-server/v2 => github.com/neilalexander/nats-server/v2 v2.8.1-0.20220419100629-2278c94774f9 +replace github.com/nats-io/nats-server/v2 => github.com/neilalexander/nats-server/v2 v2.8.3-0.20220513095553-73a9a246d34f replace github.com/nats-io/nats.go => github.com/neilalexander/nats.go v1.13.1-0.20220419101051-b262d9f0be1e @@ -30,7 +30,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20220509120958-8d818048c34c + github.com/matrix-org/gomatrixserverlib v0.0.0-20220513103617-eee8fd528433 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.10 diff --git a/go.sum b/go.sum index 8b518935c..145c2a04c 100644 --- a/go.sum +++ b/go.sum @@ -795,8 +795,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220509120958-8d818048c34c h1:KqzqFWxvs90pcDaW9QEveW+Q5JcEYuNnKyaqXc+ohno= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220509120958-8d818048c34c/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220513103617-eee8fd528433 h1:nwAlThHGPI2EAAJklXvgMcdhXF6ZiHp60+fmaYMoaDA= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220513103617-eee8fd528433/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48= github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE= github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= @@ -889,8 +889,8 @@ github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uY github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= -github.com/neilalexander/nats-server/v2 v2.8.1-0.20220419100629-2278c94774f9 h1:VGU5HYAwy8LRbSkrT+kCHvujVmwK8Aa/vc1O+eReTbM= -github.com/neilalexander/nats-server/v2 v2.8.1-0.20220419100629-2278c94774f9/go.mod h1:5vic7C58BFEVltiZhs7Kq81q2WcEPhJPsmNv1FOrdv0= +github.com/neilalexander/nats-server/v2 v2.8.3-0.20220513095553-73a9a246d34f h1:Fc+TjdV1mOy0oISSzfoxNWdTqjg7tN/Vdgf+B2cwvdo= +github.com/neilalexander/nats-server/v2 v2.8.3-0.20220513095553-73a9a246d34f/go.mod h1:vIdpKz3OG+DCg4q/xVPdXHoztEyKDWRtykQ4N7hd7C4= github.com/neilalexander/nats.go v1.13.1-0.20220419101051-b262d9f0be1e h1:kNIzIzj2OvnlreA+sTJ12nWJzTP3OSLNKDL/Iq9mF6Y= github.com/neilalexander/nats.go v1.13.1-0.20220419101051-b262d9f0be1e/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 h1:lrVQzBtkeQEGGYUHwSX1XPe1E5GL6U3KYCNe2G4bncQ= @@ -1280,7 +1280,7 @@ golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 h1:NvGWuYG8dkDHFSKksI1P9faiVJ9rayE6l0+ouWVIDs8= golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= diff --git a/internal/caching/cache_typing_test.go b/internal/caching/cache_typing_test.go index c03d89bc3..2cef32d3e 100644 --- a/internal/caching/cache_typing_test.go +++ b/internal/caching/cache_typing_test.go @@ -20,7 +20,7 @@ import ( "testing" "time" - "github.com/matrix-org/dendrite/internal/test" + "github.com/matrix-org/dendrite/test" ) func TestEDUCache(t *testing.T) { diff --git a/internal/pushgateway/client.go b/internal/pushgateway/client.go index 231327a1e..95f5afd90 100644 --- a/internal/pushgateway/client.go +++ b/internal/pushgateway/client.go @@ -3,28 +3,32 @@ package pushgateway import ( "bytes" "context" + "crypto/tls" "encoding/json" "fmt" "net/http" "time" - "github.com/matrix-org/gomatrixserverlib" "github.com/opentracing/opentracing-go" ) type httpClient struct { - hc *gomatrixserverlib.Client + hc *http.Client } // NewHTTPClient creates a new Push Gateway client. func NewHTTPClient(disableTLSValidation bool) Client { - return &httpClient{ - hc: gomatrixserverlib.NewClient( - gomatrixserverlib.WithTimeout(time.Second*30), - gomatrixserverlib.WithKeepAlives(false), - gomatrixserverlib.WithSkipVerify(disableTLSValidation), - ), + hc := &http.Client{ + Timeout: 30 * time.Second, + Transport: &http.Transport{ + DisableKeepAlives: true, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: disableTLSValidation, + }, + Proxy: http.ProxyFromEnvironment, + }, } + return &httpClient{hc: hc} } func (h *httpClient) Notify(ctx context.Context, url string, req *NotifyRequest, resp *NotifyResponse) error { @@ -41,7 +45,7 @@ func (h *httpClient) Notify(ctx context.Context, url string, req *NotifyRequest, } hreq.Header.Set("Content-Type", "application/json") - hresp, err := h.hc.DoHTTPRequest(ctx, hreq) + hresp, err := h.hc.Do(hreq) if err != nil { return err } diff --git a/internal/test/client.go b/internal/test/client.go deleted file mode 100644 index a38540ac9..000000000 --- a/internal/test/client.go +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package test - -import ( - "crypto/tls" - "fmt" - "io" - "io/ioutil" - "net/http" - "sync" - "time" - - "github.com/matrix-org/gomatrixserverlib" -) - -// Request contains the information necessary to issue a request and test its result -type Request struct { - Req *http.Request - WantedBody string - WantedStatusCode int - LastErr *LastRequestErr -} - -// LastRequestErr is a synchronised error wrapper -// Useful for obtaining the last error from a set of requests -type LastRequestErr struct { - sync.Mutex - Err error -} - -// Set sets the error -func (r *LastRequestErr) Set(err error) { - r.Lock() - defer r.Unlock() - r.Err = err -} - -// Get gets the error -func (r *LastRequestErr) Get() error { - r.Lock() - defer r.Unlock() - return r.Err -} - -// CanonicalJSONInput canonicalises a slice of JSON strings -// Useful for test input -func CanonicalJSONInput(jsonData []string) []string { - for i := range jsonData { - jsonBytes, err := gomatrixserverlib.CanonicalJSON([]byte(jsonData[i])) - if err != nil && err != io.EOF { - panic(err) - } - jsonData[i] = string(jsonBytes) - } - return jsonData -} - -// Do issues a request and checks the status code and body of the response -func (r *Request) Do() (err error) { - client := &http.Client{ - Timeout: 5 * time.Second, - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - }, - } - res, err := client.Do(r.Req) - if err != nil { - return err - } - defer (func() { err = res.Body.Close() })() - - if res.StatusCode != r.WantedStatusCode { - return fmt.Errorf("incorrect status code. Expected: %d Got: %d", r.WantedStatusCode, res.StatusCode) - } - - if r.WantedBody != "" { - resBytes, err := ioutil.ReadAll(res.Body) - if err != nil { - return err - } - jsonBytes, err := gomatrixserverlib.CanonicalJSON(resBytes) - if err != nil { - return err - } - if string(jsonBytes) != r.WantedBody { - return fmt.Errorf("returned wrong bytes. Expected:\n%s\n\nGot:\n%s", r.WantedBody, string(jsonBytes)) - } - } - - return nil -} - -// DoUntilSuccess blocks and repeats the same request until the response returns the desired status code and body. -// It then closes the given channel and returns. -func (r *Request) DoUntilSuccess(done chan error) { - r.LastErr = &LastRequestErr{} - for { - if err := r.Do(); err != nil { - r.LastErr.Set(err) - time.Sleep(1 * time.Second) // don't tightloop - continue - } - close(done) - return - } -} - -// Run repeatedly issues a request until success, error or a timeout is reached -func (r *Request) Run(label string, timeout time.Duration, serverCmdChan chan error) { - fmt.Printf("==TESTING== %v (timeout: %v)\n", label, timeout) - done := make(chan error, 1) - - // We need to wait for the server to: - // - have connected to the database - // - have created the tables - // - be listening on the given port - go r.DoUntilSuccess(done) - - // wait for one of: - // - the test to pass (done channel is closed) - // - the server to exit with an error (error sent on serverCmdChan) - // - our test timeout to expire - // We don't need to clean up since the main() function handles that in the event we panic - select { - case <-time.After(timeout): - fmt.Printf("==TESTING== %v TIMEOUT\n", label) - if reqErr := r.LastErr.Get(); reqErr != nil { - fmt.Println("Last /sync request error:") - fmt.Println(reqErr) - } - panic(fmt.Sprintf("%v server timed out", label)) - case err := <-serverCmdChan: - if err != nil { - fmt.Println("=============================================================================================") - fmt.Printf("%v server failed to run. If failing with 'pq: password authentication failed for user' try:", label) - fmt.Println(" export PGHOST=/var/run/postgresql") - fmt.Println("=============================================================================================") - panic(err) - } - case <-done: - fmt.Printf("==TESTING== %v PASSED\n", label) - } -} diff --git a/internal/test/kafka.go b/internal/test/kafka.go deleted file mode 100644 index cbf246304..000000000 --- a/internal/test/kafka.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package test - -import ( - "io" - "os/exec" - "path/filepath" - "strings" -) - -// KafkaExecutor executes kafka scripts. -type KafkaExecutor struct { - // The location of Zookeeper. Typically this is `localhost:2181`. - ZookeeperURI string - // The directory where Kafka is installed to. Used to locate kafka scripts. - KafkaDirectory string - // The location of the Kafka logs. Typically this is `localhost:9092`. - KafkaURI string - // Where stdout and stderr should be written to. Typically this is `os.Stderr`. - OutputWriter io.Writer -} - -// CreateTopic creates a new kafka topic. This is created with a single partition. -func (e *KafkaExecutor) CreateTopic(topic string) error { - cmd := exec.Command( - filepath.Join(e.KafkaDirectory, "bin", "kafka-topics.sh"), - "--create", - "--zookeeper", e.ZookeeperURI, - "--replication-factor", "1", - "--partitions", "1", - "--topic", topic, - ) - cmd.Stdout = e.OutputWriter - cmd.Stderr = e.OutputWriter - return cmd.Run() -} - -// WriteToTopic writes data to a kafka topic. -func (e *KafkaExecutor) WriteToTopic(topic string, data []string) error { - cmd := exec.Command( - filepath.Join(e.KafkaDirectory, "bin", "kafka-console-producer.sh"), - "--broker-list", e.KafkaURI, - "--topic", topic, - ) - cmd.Stdout = e.OutputWriter - cmd.Stderr = e.OutputWriter - cmd.Stdin = strings.NewReader(strings.Join(data, "\n")) - return cmd.Run() -} - -// DeleteTopic deletes a given kafka topic if it exists. -func (e *KafkaExecutor) DeleteTopic(topic string) error { - cmd := exec.Command( - filepath.Join(e.KafkaDirectory, "bin", "kafka-topics.sh"), - "--delete", - "--if-exists", - "--zookeeper", e.ZookeeperURI, - "--topic", topic, - ) - cmd.Stderr = e.OutputWriter - cmd.Stdout = e.OutputWriter - return cmd.Run() -} diff --git a/internal/test/server.go b/internal/test/server.go deleted file mode 100644 index ca14ea1bf..000000000 --- a/internal/test/server.go +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package test - -import ( - "context" - "fmt" - "net" - "net/http" - "os" - "os/exec" - "path/filepath" - "strings" - "sync" - "testing" - - "github.com/matrix-org/dendrite/setup/config" -) - -// Defaulting allows assignment of string variables with a fallback default value -// Useful for use with os.Getenv() for example -func Defaulting(value, defaultValue string) string { - if value == "" { - value = defaultValue - } - return value -} - -// CreateDatabase creates a new database, dropping it first if it exists -func CreateDatabase(command string, args []string, database string) error { - cmd := exec.Command(command, args...) - cmd.Stdin = strings.NewReader( - fmt.Sprintf("DROP DATABASE IF EXISTS %s; CREATE DATABASE %s;", database, database), - ) - // Send stdout and stderr to our stderr so that we see error messages from - // the psql process - cmd.Stdout = os.Stderr - cmd.Stderr = os.Stderr - return cmd.Run() -} - -// CreateBackgroundCommand creates an executable command -// The Cmd being executed is returned. A channel is also returned, -// which will have any termination errors sent down it, followed immediately by the channel being closed. -func CreateBackgroundCommand(command string, args []string) (*exec.Cmd, chan error) { - cmd := exec.Command(command, args...) - cmd.Stderr = os.Stderr - cmd.Stdout = os.Stderr - - if err := cmd.Start(); err != nil { - panic("failed to start server: " + err.Error()) - } - cmdChan := make(chan error, 1) - go func() { - cmdChan <- cmd.Wait() - close(cmdChan) - }() - return cmd, cmdChan -} - -// InitDatabase creates the database and config file needed for the server to run -func InitDatabase(postgresDatabase, postgresContainerName string, databases []string) { - if len(databases) > 0 { - var dbCmd string - var dbArgs []string - if postgresContainerName == "" { - dbCmd = "psql" - dbArgs = []string{postgresDatabase} - } else { - dbCmd = "docker" - dbArgs = []string{ - "exec", "-i", postgresContainerName, "psql", "-U", "postgres", postgresDatabase, - } - } - for _, database := range databases { - if err := CreateDatabase(dbCmd, dbArgs, database); err != nil { - panic(err) - } - } - } -} - -// StartProxy creates a reverse proxy -func StartProxy(bindAddr string, cfg *config.Dendrite) (*exec.Cmd, chan error) { - proxyArgs := []string{ - "--bind-address", bindAddr, - "--sync-api-server-url", "http://" + string(cfg.SyncAPI.InternalAPI.Connect), - "--client-api-server-url", "http://" + string(cfg.ClientAPI.InternalAPI.Connect), - "--media-api-server-url", "http://" + string(cfg.MediaAPI.InternalAPI.Connect), - "--tls-cert", "server.crt", - "--tls-key", "server.key", - } - return CreateBackgroundCommand( - filepath.Join(filepath.Dir(os.Args[0]), "client-api-proxy"), - proxyArgs, - ) -} - -// ListenAndServe will listen on a random high-numbered port and attach the given router. -// Returns the base URL to send requests to. Call `cancel` to shutdown the server, which will block until it has closed. -func ListenAndServe(t *testing.T, router http.Handler, useTLS bool) (apiURL string, cancel func()) { - listener, err := net.Listen("tcp", ":0") - if err != nil { - t.Fatalf("failed to listen: %s", err) - } - port := listener.Addr().(*net.TCPAddr).Port - srv := http.Server{} - - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - srv.Handler = router - var err error - if useTLS { - certFile := filepath.Join(os.TempDir(), "dendrite.cert") - keyFile := filepath.Join(os.TempDir(), "dendrite.key") - err = NewTLSKey(keyFile, certFile) - if err != nil { - t.Logf("failed to generate tls key/cert: %s", err) - return - } - err = srv.ServeTLS(listener, certFile, keyFile) - } else { - err = srv.Serve(listener) - } - if err != nil && err != http.ErrServerClosed { - t.Logf("Listen failed: %s", err) - } - }() - - secure := "" - if useTLS { - secure = "s" - } - return fmt.Sprintf("http%s://localhost:%d", secure, port), func() { - _ = srv.Shutdown(context.Background()) - wg.Wait() - } -} diff --git a/internal/version.go b/internal/version.go index e74548831..04c9a8a88 100644 --- a/internal/version.go +++ b/internal/version.go @@ -17,8 +17,8 @@ var build string const ( VersionMajor = 0 VersionMinor = 8 - VersionPatch = 3 - VersionTag = "rc1" // example: "rc1" + VersionPatch = 5 + VersionTag = "" // example: "rc1" ) func VersionString() string { diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go index 561c9a163..23f3e1a67 100644 --- a/keyserver/internal/device_list_update.go +++ b/keyserver/internal/device_list_update.go @@ -84,7 +84,7 @@ type DeviceListUpdater struct { db DeviceListUpdaterDatabase api DeviceListUpdaterAPI producer KeyChangeProducer - fedClient fedsenderapi.FederationClient + fedClient fedsenderapi.KeyserverFederationAPI workerChans []chan gomatrixserverlib.ServerName // When device lists are stale for a user, they get inserted into this map with a channel which `Update` will @@ -127,7 +127,7 @@ type KeyChangeProducer interface { // NewDeviceListUpdater creates a new updater which fetches fresh device lists when they go stale. func NewDeviceListUpdater( db DeviceListUpdaterDatabase, api DeviceListUpdaterAPI, producer KeyChangeProducer, - fedClient fedsenderapi.FederationClient, numWorkers int, + fedClient fedsenderapi.KeyserverFederationAPI, numWorkers int, ) *DeviceListUpdater { return &DeviceListUpdater{ userIDToMutex: make(map[string]*sync.Mutex), diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index be71e5750..f8d0d69c3 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -37,7 +37,7 @@ import ( type KeyInternalAPI struct { DB storage.Database ThisServer gomatrixserverlib.ServerName - FedClient fedsenderapi.FederationClient + FedClient fedsenderapi.KeyserverFederationAPI UserAPI userapi.KeyserverUserAPI Producer *producers.KeyChange Updater *DeviceListUpdater diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go index 47d7f57f9..3ffd3ba1e 100644 --- a/keyserver/keyserver.go +++ b/keyserver/keyserver.go @@ -37,7 +37,7 @@ func AddInternalRoutes(router *mux.Router, intAPI api.KeyInternalAPI) { // NewInternalAPI returns a concerete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( - base *base.BaseDendrite, cfg *config.KeyServer, fedClient fedsenderapi.FederationClient, + base *base.BaseDendrite, cfg *config.KeyServer, fedClient fedsenderapi.KeyserverFederationAPI, ) api.KeyInternalAPI { js, _ := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream) diff --git a/keyserver/storage/postgres/one_time_keys_table.go b/keyserver/storage/postgres/one_time_keys_table.go index d8c76b49b..2117efcae 100644 --- a/keyserver/storage/postgres/one_time_keys_table.go +++ b/keyserver/storage/postgres/one_time_keys_table.go @@ -53,7 +53,9 @@ const selectKeysSQL = "" + "SELECT concat(algorithm, ':', key_id) as algorithmwithid, key_json FROM keyserver_one_time_keys WHERE user_id=$1 AND device_id=$2 AND concat(algorithm, ':', key_id) = ANY($3);" const selectKeysCountSQL = "" + - "SELECT algorithm, COUNT(key_id) FROM keyserver_one_time_keys WHERE user_id=$1 AND device_id=$2 GROUP BY algorithm" + "SELECT algorithm, COUNT(key_id) FROM " + + " (SELECT algorithm, key_id FROM keyserver_one_time_keys WHERE user_id = $1 AND device_id = $2 LIMIT 100)" + + " x GROUP BY algorithm" const deleteOneTimeKeySQL = "" + "DELETE FROM keyserver_one_time_keys WHERE user_id = $1 AND device_id = $2 AND algorithm = $3 AND key_id = $4" diff --git a/keyserver/storage/sqlite3/one_time_keys_table.go b/keyserver/storage/sqlite3/one_time_keys_table.go index d2c0b7b20..7a923d0e5 100644 --- a/keyserver/storage/sqlite3/one_time_keys_table.go +++ b/keyserver/storage/sqlite3/one_time_keys_table.go @@ -52,7 +52,9 @@ const selectKeysSQL = "" + "SELECT key_id, algorithm, key_json FROM keyserver_one_time_keys WHERE user_id=$1 AND device_id=$2" const selectKeysCountSQL = "" + - "SELECT algorithm, COUNT(key_id) FROM keyserver_one_time_keys WHERE user_id=$1 AND device_id=$2 GROUP BY algorithm" + "SELECT algorithm, COUNT(key_id) FROM " + + " (SELECT algorithm, key_id FROM keyserver_one_time_keys WHERE user_id = $1 AND device_id = $2 LIMIT 100)" + + " x GROUP BY algorithm" const deleteOneTimeKeySQL = "" + "DELETE FROM keyserver_one_time_keys WHERE user_id = $1 AND device_id = $2 AND algorithm = $3 AND key_id = $4" diff --git a/mediaapi/thumbnailer/thumbnailer_bimg.go b/mediaapi/thumbnailer/thumbnailer_bimg.go index 6ca533176..fa1acbf08 100644 --- a/mediaapi/thumbnailer/thumbnailer_bimg.go +++ b/mediaapi/thumbnailer/thumbnailer_bimg.go @@ -37,7 +37,7 @@ func GenerateThumbnails( mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, - db *storage.Database, + db storage.Database, logger *log.Entry, ) (busy bool, errorReturn error) { buffer, err := bimg.Read(string(src)) @@ -49,7 +49,7 @@ func GenerateThumbnails( for _, config := range configs { // Note: createThumbnail does locking based on activeThumbnailGeneration busy, err = createThumbnail( - ctx, src, img, config, mediaMetadata, activeThumbnailGeneration, + ctx, src, img, types.ThumbnailSize(config), mediaMetadata, activeThumbnailGeneration, maxThumbnailGenerators, db, logger, ) if err != nil { @@ -71,7 +71,7 @@ func GenerateThumbnail( mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, - db *storage.Database, + db storage.Database, logger *log.Entry, ) (busy bool, errorReturn error) { buffer, err := bimg.Read(string(src)) @@ -109,7 +109,7 @@ func createThumbnail( mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, - db *storage.Database, + db storage.Database, logger *log.Entry, ) (busy bool, errorReturn error) { logger = logger.WithFields(log.Fields{ diff --git a/roomserver/api/api.go b/roomserver/api/api.go index cbb4cebca..80e7aed64 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -183,6 +183,7 @@ type FederationRoomserverAPI interface { QueryMissingEvents(ctx context.Context, req *QueryMissingEventsRequest, res *QueryMissingEventsResponse) error // Query whether a server is allowed to see an event QueryServerAllowedToSeeEvent(ctx context.Context, req *QueryServerAllowedToSeeEventRequest, res *QueryServerAllowedToSeeEventResponse) error + QueryRoomsForUser(ctx context.Context, req *QueryRoomsForUserRequest, res *QueryRoomsForUserResponse) error PerformInboundPeek(ctx context.Context, req *PerformInboundPeekRequest, res *PerformInboundPeekResponse) error PerformInvite(ctx context.Context, req *PerformInviteRequest, res *PerformInviteResponse) error // Query a given amount (or less) of events prior to a given set of events. diff --git a/roomserver/api/query.go b/roomserver/api/query.go index ef2e6bb57..afafb87c3 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -122,6 +122,7 @@ type QueryMembershipForUserResponse struct { Membership string `json:"membership"` // True if the user asked to forget this room. IsRoomForgotten bool `json:"is_room_forgotten"` + RoomExists bool `json:"room_exists"` } // QueryMembershipsForRoomRequest is a request to QueryMembershipsForRoom diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index 9ad8b0422..e4c138d58 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -233,12 +233,19 @@ func (u *latestEventsUpdater) latestState() error { } } - // Get a list of the current latest events. This may or may not - // include the new event from the input path, depending on whether - // it is a forward extremity or not. - latestStateAtEvents := make([]types.StateAtEvent, len(u.latest)) - for i := range u.latest { - latestStateAtEvents[i] = u.latest[i].StateAtEvent + // Take the old set of extremities and the new set of extremities and + // mash them together into a list. This may or may not include the new event + // from the input path, depending on whether it became a forward extremity + // or not. We'll then run state resolution across all of them to determine + // the new current state of the room. Including the old extremities here + // ensures that new forward extremities with bad state snapshots (from + // possible malicious actors) can't completely corrupt the room state + // away from what it was before. + combinedExtremities := types.StateAtEventAndReferences(append(u.oldLatest, u.latest...)) + combinedExtremities = combinedExtremities[:util.SortAndUnique(combinedExtremities)] + latestStateAtEvents := make([]types.StateAtEvent, len(combinedExtremities)) + for i := range combinedExtremities { + latestStateAtEvents[i] = combinedExtremities[i].StateAtEvent } // Takes the NIDs of the latest events and creates a state snapshot diff --git a/roomserver/internal/input/input_test.go b/roomserver/internal/input/input_test.go index a95c13550..7c65f9eac 100644 --- a/roomserver/internal/input/input_test.go +++ b/roomserver/internal/input/input_test.go @@ -12,7 +12,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/gomatrixserverlib" "github.com/nats-io/nats.go" ) @@ -22,7 +22,7 @@ var jc *nats.Conn func TestMain(m *testing.M) { var b *base.BaseDendrite - b, js, jc = test.Base(nil) + b, js, jc = testrig.Base(nil) code := m.Run() b.ShutdownDendrite() b.WaitForComponentsToFinish() diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index 5b33ec3c3..d25bdc378 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -169,8 +169,10 @@ func (r *Queryer) QueryMembershipForUser( return err } if info == nil { - return fmt.Errorf("QueryMembershipForUser: unknown room %s", request.RoomID) + response.RoomExists = false + return nil } + response.RoomExists = true membershipEventNID, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, request.UserID) if err != nil { diff --git a/roomserver/internal/query/query_test.go b/roomserver/internal/query/query_test.go index ba5bb9f55..03627ea97 100644 --- a/roomserver/internal/query/query_test.go +++ b/roomserver/internal/query/query_test.go @@ -19,8 +19,8 @@ import ( "encoding/json" "testing" - "github.com/matrix-org/dendrite/internal/test" "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/dendrite/test" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go index 86d226ce7..a4d05756d 100644 --- a/roomserver/storage/postgres/events_table.go +++ b/roomserver/storage/postgres/events_table.go @@ -264,11 +264,11 @@ func (s *eventStatements) BulkSelectStateEventByNID( ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID, stateKeyTuples []types.StateKeyTuple, ) ([]types.StateEntry, error) { - tuples := stateKeyTupleSorter(stateKeyTuples) + tuples := types.StateKeyTupleSorter(stateKeyTuples) sort.Sort(tuples) - eventTypeNIDArray, eventStateKeyNIDArray := tuples.typesAndStateKeysAsArrays() + eventTypeNIDArray, eventStateKeyNIDArray := tuples.TypesAndStateKeysAsArrays() stmt := sqlutil.TxStmt(txn, s.bulkSelectStateEventByNIDStmt) - rows, err := stmt.QueryContext(ctx, eventNIDsAsArray(eventNIDs), eventTypeNIDArray, eventStateKeyNIDArray) + rows, err := stmt.QueryContext(ctx, eventNIDsAsArray(eventNIDs), pq.Int64Array(eventTypeNIDArray), pq.Int64Array(eventStateKeyNIDArray)) if err != nil { return nil, err } diff --git a/roomserver/storage/postgres/invite_table.go b/roomserver/storage/postgres/invite_table.go index 176c16e48..4cddfe2e9 100644 --- a/roomserver/storage/postgres/invite_table.go +++ b/roomserver/storage/postgres/invite_table.go @@ -81,12 +81,12 @@ type inviteStatements struct { updateInviteRetiredStmt *sql.Stmt } -func createInvitesTable(db *sql.DB) error { +func CreateInvitesTable(db *sql.DB) error { _, err := db.Exec(inviteSchema) return err } -func prepareInvitesTable(db *sql.DB) (tables.Invites, error) { +func PrepareInvitesTable(db *sql.DB) (tables.Invites, error) { s := &inviteStatements{} return s, sqlutil.StatementList{ @@ -127,8 +127,8 @@ func (s *inviteStatements) UpdateInviteRetired( defer internal.CloseAndLogIfError(ctx, rows, "updateInviteRetired: rows.close() failed") var eventIDs []string + var inviteEventID string for rows.Next() { - var inviteEventID string if err = rows.Scan(&inviteEventID); err != nil { return nil, err } @@ -152,9 +152,9 @@ func (s *inviteStatements) SelectInviteActiveForUserInRoom( defer internal.CloseAndLogIfError(ctx, rows, "selectInviteActiveForUserInRoom: rows.close() failed") var result []types.EventStateKeyNID var eventIDs []string + var inviteEventID string + var senderUserNID int64 for rows.Next() { - var inviteEventID string - var senderUserNID int64 if err := rows.Scan(&inviteEventID, &senderUserNID); err != nil { return nil, nil, err } diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go index 6ed5293e4..c01753c3a 100644 --- a/roomserver/storage/postgres/membership_table.go +++ b/roomserver/storage/postgres/membership_table.go @@ -160,12 +160,12 @@ type membershipStatements struct { selectServerInRoomStmt *sql.Stmt } -func createMembershipTable(db *sql.DB) error { +func CreateMembershipTable(db *sql.DB) error { _, err := db.Exec(membershipSchema) return err } -func prepareMembershipTable(db *sql.DB) (tables.Membership, error) { +func PrepareMembershipTable(db *sql.DB) (tables.Membership, error) { s := &membershipStatements{} return s, sqlutil.StatementList{ @@ -234,8 +234,8 @@ func (s *membershipStatements) SelectMembershipsFromRoom( } defer internal.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoom: rows.close() failed") + var eNID types.EventNID for rows.Next() { - var eNID types.EventNID if err = rows.Scan(&eNID); err != nil { return } @@ -262,8 +262,8 @@ func (s *membershipStatements) SelectMembershipsFromRoomAndMembership( } defer internal.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoomAndMembership: rows.close() failed") + var eNID types.EventNID for rows.Next() { - var eNID types.EventNID if err = rows.Scan(&eNID); err != nil { return } @@ -298,8 +298,8 @@ func (s *membershipStatements) SelectRoomsWithMembership( } defer internal.CloseAndLogIfError(ctx, rows, "SelectRoomsWithMembership: rows.close() failed") var roomNIDs []types.RoomNID + var roomNID types.RoomNID for rows.Next() { - var roomNID types.RoomNID if err := rows.Scan(&roomNID); err != nil { return nil, err } @@ -320,9 +320,9 @@ func (s *membershipStatements) SelectJoinedUsersSetForRooms( } defer internal.CloseAndLogIfError(ctx, rows, "selectJoinedUsersSetForRooms: rows.close() failed") result := make(map[types.EventStateKeyNID]int) + var userID types.EventStateKeyNID + var count int for rows.Next() { - var userID types.EventStateKeyNID - var count int if err := rows.Scan(&userID, &count); err != nil { return nil, err } @@ -342,12 +342,12 @@ func (s *membershipStatements) SelectKnownUsers( } result := []string{} defer internal.CloseAndLogIfError(ctx, rows, "SelectKnownUsers: rows.close() failed") + var resUserID string for rows.Next() { - var userID string - if err := rows.Scan(&userID); err != nil { + if err := rows.Scan(&resUserID); err != nil { return nil, err } - result = append(result, userID) + result = append(result, resUserID) } return result, rows.Err() } diff --git a/roomserver/storage/postgres/previous_events_table.go b/roomserver/storage/postgres/previous_events_table.go index bd4e853eb..26999a290 100644 --- a/roomserver/storage/postgres/previous_events_table.go +++ b/roomserver/storage/postgres/previous_events_table.go @@ -64,12 +64,12 @@ type previousEventStatements struct { selectPreviousEventExistsStmt *sql.Stmt } -func createPrevEventsTable(db *sql.DB) error { +func CreatePrevEventsTable(db *sql.DB) error { _, err := db.Exec(previousEventSchema) return err } -func preparePrevEventsTable(db *sql.DB) (tables.PreviousEvents, error) { +func PreparePrevEventsTable(db *sql.DB) (tables.PreviousEvents, error) { s := &previousEventStatements{} return s, sqlutil.StatementList{ diff --git a/roomserver/storage/postgres/published_table.go b/roomserver/storage/postgres/published_table.go index 15985fcd6..56fa02f7b 100644 --- a/roomserver/storage/postgres/published_table.go +++ b/roomserver/storage/postgres/published_table.go @@ -49,12 +49,12 @@ type publishedStatements struct { selectPublishedStmt *sql.Stmt } -func createPublishedTable(db *sql.DB) error { +func CreatePublishedTable(db *sql.DB) error { _, err := db.Exec(publishedSchema) return err } -func preparePublishedTable(db *sql.DB) (tables.Published, error) { +func PreparePublishedTable(db *sql.DB) (tables.Published, error) { s := &publishedStatements{} return s, sqlutil.StatementList{ @@ -94,8 +94,8 @@ func (s *publishedStatements) SelectAllPublishedRooms( defer internal.CloseAndLogIfError(ctx, rows, "selectAllPublishedStmt: rows.close() failed") var roomIDs []string + var roomID string for rows.Next() { - var roomID string if err = rows.Scan(&roomID); err != nil { return nil, err } diff --git a/roomserver/storage/postgres/redactions_table.go b/roomserver/storage/postgres/redactions_table.go index 5614f2bd8..6e2f6712d 100644 --- a/roomserver/storage/postgres/redactions_table.go +++ b/roomserver/storage/postgres/redactions_table.go @@ -59,12 +59,12 @@ type redactionStatements struct { markRedactionValidatedStmt *sql.Stmt } -func createRedactionsTable(db *sql.DB) error { +func CreateRedactionsTable(db *sql.DB) error { _, err := db.Exec(redactionsSchema) return err } -func prepareRedactionsTable(db *sql.DB) (tables.Redactions, error) { +func PrepareRedactionsTable(db *sql.DB) (tables.Redactions, error) { s := &redactionStatements{} return s, sqlutil.StatementList{ diff --git a/roomserver/storage/postgres/room_aliases_table.go b/roomserver/storage/postgres/room_aliases_table.go index d13df8e7f..a84929f61 100644 --- a/roomserver/storage/postgres/room_aliases_table.go +++ b/roomserver/storage/postgres/room_aliases_table.go @@ -61,12 +61,12 @@ type roomAliasesStatements struct { deleteRoomAliasStmt *sql.Stmt } -func createRoomAliasesTable(db *sql.DB) error { +func CreateRoomAliasesTable(db *sql.DB) error { _, err := db.Exec(roomAliasesSchema) return err } -func prepareRoomAliasesTable(db *sql.DB) (tables.RoomAliases, error) { +func PrepareRoomAliasesTable(db *sql.DB) (tables.RoomAliases, error) { s := &roomAliasesStatements{} return s, sqlutil.StatementList{ @@ -108,8 +108,8 @@ func (s *roomAliasesStatements) SelectAliasesFromRoomID( defer internal.CloseAndLogIfError(ctx, rows, "selectAliasesFromRoomID: rows.close() failed") var aliases []string + var alias string for rows.Next() { - var alias string if err = rows.Scan(&alias); err != nil { return nil, err } diff --git a/roomserver/storage/postgres/rooms_table.go b/roomserver/storage/postgres/rooms_table.go index b2685084d..24362af74 100644 --- a/roomserver/storage/postgres/rooms_table.go +++ b/roomserver/storage/postgres/rooms_table.go @@ -95,12 +95,12 @@ type roomStatements struct { bulkSelectRoomNIDsStmt *sql.Stmt } -func createRoomsTable(db *sql.DB) error { +func CreateRoomsTable(db *sql.DB) error { _, err := db.Exec(roomsSchema) return err } -func prepareRoomsTable(db *sql.DB) (tables.Rooms, error) { +func PrepareRoomsTable(db *sql.DB) (tables.Rooms, error) { s := &roomStatements{} return s, sqlutil.StatementList{ @@ -117,7 +117,7 @@ func prepareRoomsTable(db *sql.DB) (tables.Rooms, error) { }.Prepare(db) } -func (s *roomStatements) SelectRoomIDs(ctx context.Context, txn *sql.Tx) ([]string, error) { +func (s *roomStatements) SelectRoomIDsWithEvents(ctx context.Context, txn *sql.Tx) ([]string, error) { stmt := sqlutil.TxStmt(txn, s.selectRoomIDsStmt) rows, err := stmt.QueryContext(ctx) if err != nil { @@ -125,8 +125,8 @@ func (s *roomStatements) SelectRoomIDs(ctx context.Context, txn *sql.Tx) ([]stri } defer internal.CloseAndLogIfError(ctx, rows, "selectRoomIDsStmt: rows.close() failed") var roomIDs []string + var roomID string for rows.Next() { - var roomID string if err = rows.Scan(&roomID); err != nil { return nil, err } @@ -231,9 +231,9 @@ func (s *roomStatements) SelectRoomVersionsForRoomNIDs( } defer internal.CloseAndLogIfError(ctx, rows, "selectRoomVersionsForRoomNIDsStmt: rows.close() failed") result := make(map[types.RoomNID]gomatrixserverlib.RoomVersion) + var roomNID types.RoomNID + var roomVersion gomatrixserverlib.RoomVersion for rows.Next() { - var roomNID types.RoomNID - var roomVersion gomatrixserverlib.RoomVersion if err = rows.Scan(&roomNID, &roomVersion); err != nil { return nil, err } @@ -254,8 +254,8 @@ func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roo } defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectRoomIDsStmt: rows.close() failed") var roomIDs []string + var roomID string for rows.Next() { - var roomID string if err = rows.Scan(&roomID); err != nil { return nil, err } @@ -276,8 +276,8 @@ func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, ro } defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectRoomNIDsStmt: rows.close() failed") var roomNIDs []types.RoomNID + var roomNID types.RoomNID for rows.Next() { - var roomNID types.RoomNID if err = rows.Scan(&roomNID); err != nil { return nil, err } diff --git a/roomserver/storage/postgres/state_block_table.go b/roomserver/storage/postgres/state_block_table.go index 6f8f9e1b5..5af48f031 100644 --- a/roomserver/storage/postgres/state_block_table.go +++ b/roomserver/storage/postgres/state_block_table.go @@ -19,7 +19,6 @@ import ( "context" "database/sql" "fmt" - "sort" "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" @@ -71,12 +70,12 @@ type stateBlockStatements struct { bulkSelectStateBlockEntriesStmt *sql.Stmt } -func createStateBlockTable(db *sql.DB) error { +func CreateStateBlockTable(db *sql.DB) error { _, err := db.Exec(stateDataSchema) return err } -func prepareStateBlockTable(db *sql.DB) (tables.StateBlock, error) { +func PrepareStateBlockTable(db *sql.DB) (tables.StateBlock, error) { s := &stateBlockStatements{} return s, sqlutil.StatementList{ @@ -90,9 +89,9 @@ func (s *stateBlockStatements) BulkInsertStateData( entries types.StateEntries, ) (id types.StateBlockNID, err error) { entries = entries[:util.SortAndUnique(entries)] - var nids types.EventNIDs - for _, e := range entries { - nids = append(nids, e.EventNID) + nids := make(types.EventNIDs, entries.Len()) + for i := range entries { + nids[i] = entries[i].EventNID } stmt := sqlutil.TxStmt(txn, s.insertStateDataStmt) err = stmt.QueryRowContext( @@ -113,15 +112,15 @@ func (s *stateBlockStatements) BulkSelectStateBlockEntries( results := make([][]types.EventNID, len(stateBlockNIDs)) i := 0 + var stateBlockNID types.StateBlockNID + var result pq.Int64Array for ; rows.Next(); i++ { - var stateBlockNID types.StateBlockNID - var result pq.Int64Array if err = rows.Scan(&stateBlockNID, &result); err != nil { return nil, err } - r := []types.EventNID{} - for _, e := range result { - r = append(r, types.EventNID(e)) + r := make([]types.EventNID, len(result)) + for x := range result { + r[x] = types.EventNID(result[x]) } results[i] = r } @@ -141,35 +140,3 @@ func stateBlockNIDsAsArray(stateBlockNIDs []types.StateBlockNID) pq.Int64Array { } return pq.Int64Array(nids) } - -type stateKeyTupleSorter []types.StateKeyTuple - -func (s stateKeyTupleSorter) Len() int { return len(s) } -func (s stateKeyTupleSorter) Less(i, j int) bool { return s[i].LessThan(s[j]) } -func (s stateKeyTupleSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -// Check whether a tuple is in the list. Assumes that the list is sorted. -func (s stateKeyTupleSorter) contains(value types.StateKeyTuple) bool { - i := sort.Search(len(s), func(i int) bool { return !s[i].LessThan(value) }) - return i < len(s) && s[i] == value -} - -// List the unique eventTypeNIDs and eventStateKeyNIDs. -// Assumes that the list is sorted. -func (s stateKeyTupleSorter) typesAndStateKeysAsArrays() (eventTypeNIDs pq.Int64Array, eventStateKeyNIDs pq.Int64Array) { - eventTypeNIDs = make(pq.Int64Array, len(s)) - eventStateKeyNIDs = make(pq.Int64Array, len(s)) - for i := range s { - eventTypeNIDs[i] = int64(s[i].EventTypeNID) - eventStateKeyNIDs[i] = int64(s[i].EventStateKeyNID) - } - eventTypeNIDs = eventTypeNIDs[:util.SortAndUnique(int64Sorter(eventTypeNIDs))] - eventStateKeyNIDs = eventStateKeyNIDs[:util.SortAndUnique(int64Sorter(eventStateKeyNIDs))] - return -} - -type int64Sorter []int64 - -func (s int64Sorter) Len() int { return len(s) } -func (s int64Sorter) Less(i, j int) bool { return s[i] < s[j] } -func (s int64Sorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } diff --git a/roomserver/storage/postgres/state_block_table_test.go b/roomserver/storage/postgres/state_block_table_test.go deleted file mode 100644 index a0e2ec952..000000000 --- a/roomserver/storage/postgres/state_block_table_test.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package postgres - -import ( - "sort" - "testing" - - "github.com/matrix-org/dendrite/roomserver/types" -) - -func TestStateKeyTupleSorter(t *testing.T) { - input := stateKeyTupleSorter{ - {EventTypeNID: 1, EventStateKeyNID: 2}, - {EventTypeNID: 1, EventStateKeyNID: 4}, - {EventTypeNID: 2, EventStateKeyNID: 2}, - {EventTypeNID: 1, EventStateKeyNID: 1}, - } - want := []types.StateKeyTuple{ - {EventTypeNID: 1, EventStateKeyNID: 1}, - {EventTypeNID: 1, EventStateKeyNID: 2}, - {EventTypeNID: 1, EventStateKeyNID: 4}, - {EventTypeNID: 2, EventStateKeyNID: 2}, - } - doNotWant := []types.StateKeyTuple{ - {EventTypeNID: 0, EventStateKeyNID: 0}, - {EventTypeNID: 1, EventStateKeyNID: 3}, - {EventTypeNID: 2, EventStateKeyNID: 1}, - {EventTypeNID: 3, EventStateKeyNID: 1}, - } - wantTypeNIDs := []int64{1, 2} - wantStateKeyNIDs := []int64{1, 2, 4} - - // Sort the input and check it's in the right order. - sort.Sort(input) - gotTypeNIDs, gotStateKeyNIDs := input.typesAndStateKeysAsArrays() - - for i := range want { - if input[i] != want[i] { - t.Errorf("Wanted %#v at index %d got %#v", want[i], i, input[i]) - } - - if !input.contains(want[i]) { - t.Errorf("Wanted %#v.contains(%#v) to be true but got false", input, want[i]) - } - } - - for i := range doNotWant { - if input.contains(doNotWant[i]) { - t.Errorf("Wanted %#v.contains(%#v) to be false but got true", input, doNotWant[i]) - } - } - - if len(wantTypeNIDs) != len(gotTypeNIDs) { - t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs) - } - - for i := range wantTypeNIDs { - if wantTypeNIDs[i] != gotTypeNIDs[i] { - t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs) - } - } - - if len(wantStateKeyNIDs) != len(gotStateKeyNIDs) { - t.Fatalf("Wanted state key NIDs %#v got %#v", wantStateKeyNIDs, gotStateKeyNIDs) - } - - for i := range wantStateKeyNIDs { - if wantStateKeyNIDs[i] != gotStateKeyNIDs[i] { - t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs) - } - } -} diff --git a/roomserver/storage/postgres/state_snapshot_table.go b/roomserver/storage/postgres/state_snapshot_table.go index 8ed886030..a24b7f3f0 100644 --- a/roomserver/storage/postgres/state_snapshot_table.go +++ b/roomserver/storage/postgres/state_snapshot_table.go @@ -77,12 +77,12 @@ type stateSnapshotStatements struct { bulkSelectStateBlockNIDsStmt *sql.Stmt } -func createStateSnapshotTable(db *sql.DB) error { +func CreateStateSnapshotTable(db *sql.DB) error { _, err := db.Exec(stateSnapshotSchema) return err } -func prepareStateSnapshotTable(db *sql.DB) (tables.StateSnapshot, error) { +func PrepareStateSnapshotTable(db *sql.DB) (tables.StateSnapshot, error) { s := &stateSnapshotStatements{} return s, sqlutil.StatementList{ @@ -95,12 +95,10 @@ func (s *stateSnapshotStatements) InsertState( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, nids types.StateBlockNIDs, ) (stateNID types.StateSnapshotNID, err error) { nids = nids[:util.SortAndUnique(nids)] - var id int64 - err = sqlutil.TxStmt(txn, s.insertStateStmt).QueryRowContext(ctx, nids.Hash(), int64(roomNID), stateBlockNIDsAsArray(nids)).Scan(&id) + err = sqlutil.TxStmt(txn, s.insertStateStmt).QueryRowContext(ctx, nids.Hash(), int64(roomNID), stateBlockNIDsAsArray(nids)).Scan(&stateNID) if err != nil { return 0, err } - stateNID = types.StateSnapshotNID(id) return } @@ -119,9 +117,9 @@ func (s *stateSnapshotStatements) BulkSelectStateBlockNIDs( defer rows.Close() // nolint: errcheck results := make([]types.StateBlockNIDList, len(stateNIDs)) i := 0 + var stateBlockNIDs pq.Int64Array for ; rows.Next(); i++ { result := &results[i] - var stateBlockNIDs pq.Int64Array if err = rows.Scan(&result.StateSnapshotNID, &stateBlockNIDs); err != nil { return nil, err } diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index 34e891490..70ea4d8ba 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -80,31 +80,31 @@ func (d *Database) create(db *sql.DB) error { if err := CreateEventsTable(db); err != nil { return err } - if err := createRoomsTable(db); err != nil { + if err := CreateRoomsTable(db); err != nil { return err } - if err := createStateBlockTable(db); err != nil { + if err := CreateStateBlockTable(db); err != nil { return err } - if err := createStateSnapshotTable(db); err != nil { + if err := CreateStateSnapshotTable(db); err != nil { return err } - if err := createPrevEventsTable(db); err != nil { + if err := CreatePrevEventsTable(db); err != nil { return err } - if err := createRoomAliasesTable(db); err != nil { + if err := CreateRoomAliasesTable(db); err != nil { return err } - if err := createInvitesTable(db); err != nil { + if err := CreateInvitesTable(db); err != nil { return err } - if err := createMembershipTable(db); err != nil { + if err := CreateMembershipTable(db); err != nil { return err } - if err := createPublishedTable(db); err != nil { + if err := CreatePublishedTable(db); err != nil { return err } - if err := createRedactionsTable(db); err != nil { + if err := CreateRedactionsTable(db); err != nil { return err } @@ -128,39 +128,39 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room if err != nil { return err } - rooms, err := prepareRoomsTable(db) + rooms, err := PrepareRoomsTable(db) if err != nil { return err } - stateBlock, err := prepareStateBlockTable(db) + stateBlock, err := PrepareStateBlockTable(db) if err != nil { return err } - stateSnapshot, err := prepareStateSnapshotTable(db) + stateSnapshot, err := PrepareStateSnapshotTable(db) if err != nil { return err } - prevEvents, err := preparePrevEventsTable(db) + prevEvents, err := PreparePrevEventsTable(db) if err != nil { return err } - roomAliases, err := prepareRoomAliasesTable(db) + roomAliases, err := PrepareRoomAliasesTable(db) if err != nil { return err } - invites, err := prepareInvitesTable(db) + invites, err := PrepareInvitesTable(db) if err != nil { return err } - membership, err := prepareMembershipTable(db) + membership, err := PrepareMembershipTable(db) if err != nil { return err } - published, err := preparePublishedTable(db) + published, err := PreparePublishedTable(db) if err != nil { return err } - redactions, err := prepareRedactionsTable(db) + redactions, err := PrepareRedactionsTable(db) if err != nil { return err } diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index 252e94c7e..cc4a9fff5 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -1216,7 +1216,7 @@ func (d *Database) GetKnownUsers(ctx context.Context, userID, searchString strin // GetKnownRooms returns a list of all rooms we know about. func (d *Database) GetKnownRooms(ctx context.Context) ([]string, error) { - return d.RoomsTable.SelectRoomIDs(ctx, nil) + return d.RoomsTable.SelectRoomIDsWithEvents(ctx, nil) } // ForgetRoom sets a users room to forgotten diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index feb06150a..1dda34c36 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -247,9 +247,9 @@ func (s *eventStatements) BulkSelectStateEventByNID( ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID, stateKeyTuples []types.StateKeyTuple, ) ([]types.StateEntry, error) { - tuples := stateKeyTupleSorter(stateKeyTuples) + tuples := types.StateKeyTupleSorter(stateKeyTuples) sort.Sort(tuples) - eventTypeNIDArray, eventStateKeyNIDArray := tuples.typesAndStateKeysAsArrays() + eventTypeNIDArray, eventStateKeyNIDArray := tuples.TypesAndStateKeysAsArrays() params := make([]interface{}, 0, len(eventNIDs)+len(eventTypeNIDArray)+len(eventStateKeyNIDArray)) selectOrig := strings.Replace(bulkSelectStateEventByNIDSQL, "($1)", sqlutil.QueryVariadic(len(eventNIDs)), 1) for _, v := range eventNIDs { diff --git a/roomserver/storage/sqlite3/invite_table.go b/roomserver/storage/sqlite3/invite_table.go index d54d313a9..e051d63af 100644 --- a/roomserver/storage/sqlite3/invite_table.go +++ b/roomserver/storage/sqlite3/invite_table.go @@ -69,12 +69,12 @@ type inviteStatements struct { selectInvitesAboutToRetireStmt *sql.Stmt } -func createInvitesTable(db *sql.DB) error { +func CreateInvitesTable(db *sql.DB) error { _, err := db.Exec(inviteSchema) return err } -func prepareInvitesTable(db *sql.DB) (tables.Invites, error) { +func PrepareInvitesTable(db *sql.DB) (tables.Invites, error) { s := &inviteStatements{ db: db, } @@ -119,8 +119,8 @@ func (s *inviteStatements) UpdateInviteRetired( return } defer internal.CloseAndLogIfError(ctx, rows, "UpdateInviteRetired: rows.close() failed") + var inviteEventID string for rows.Next() { - var inviteEventID string if err = rows.Scan(&inviteEventID); err != nil { return } @@ -147,9 +147,9 @@ func (s *inviteStatements) SelectInviteActiveForUserInRoom( defer internal.CloseAndLogIfError(ctx, rows, "selectInviteActiveForUserInRoom: rows.close() failed") var result []types.EventStateKeyNID var eventIDs []string + var eventID string + var senderUserNID int64 for rows.Next() { - var eventID string - var senderUserNID int64 if err := rows.Scan(&eventID, &senderUserNID); err != nil { return nil, nil, err } diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go index 7ed86b612..6f0fe8b64 100644 --- a/roomserver/storage/sqlite3/membership_table.go +++ b/roomserver/storage/sqlite3/membership_table.go @@ -136,12 +136,12 @@ type membershipStatements struct { selectServerInRoomStmt *sql.Stmt } -func createMembershipTable(db *sql.DB) error { +func CreateMembershipTable(db *sql.DB) error { _, err := db.Exec(membershipSchema) return err } -func prepareMembershipTable(db *sql.DB) (tables.Membership, error) { +func PrepareMembershipTable(db *sql.DB) (tables.Membership, error) { s := &membershipStatements{ db: db, } @@ -212,8 +212,8 @@ func (s *membershipStatements) SelectMembershipsFromRoom( } defer internal.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoom: rows.close() failed") + var eNID types.EventNID for rows.Next() { - var eNID types.EventNID if err = rows.Scan(&eNID); err != nil { return } @@ -239,8 +239,8 @@ func (s *membershipStatements) SelectMembershipsFromRoomAndMembership( } defer internal.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoomAndMembership: rows.close() failed") + var eNID types.EventNID for rows.Next() { - var eNID types.EventNID if err = rows.Scan(&eNID); err != nil { return } @@ -275,8 +275,8 @@ func (s *membershipStatements) SelectRoomsWithMembership( } defer internal.CloseAndLogIfError(ctx, rows, "SelectRoomsWithMembership: rows.close() failed") var roomNIDs []types.RoomNID + var roomNID types.RoomNID for rows.Next() { - var roomNID types.RoomNID if err := rows.Scan(&roomNID); err != nil { return nil, err } @@ -307,9 +307,9 @@ func (s *membershipStatements) SelectJoinedUsersSetForRooms(ctx context.Context, } defer internal.CloseAndLogIfError(ctx, rows, "selectJoinedUsersSetForRooms: rows.close() failed") result := make(map[types.EventStateKeyNID]int) + var userID types.EventStateKeyNID + var count int for rows.Next() { - var userID types.EventStateKeyNID - var count int if err := rows.Scan(&userID, &count); err != nil { return nil, err } @@ -326,12 +326,12 @@ func (s *membershipStatements) SelectKnownUsers(ctx context.Context, txn *sql.Tx } result := []string{} defer internal.CloseAndLogIfError(ctx, rows, "SelectKnownUsers: rows.close() failed") + var resUserID string for rows.Next() { - var userID string - if err := rows.Scan(&userID); err != nil { + if err := rows.Scan(&resUserID); err != nil { return nil, err } - result = append(result, userID) + result = append(result, resUserID) } return result, rows.Err() } diff --git a/roomserver/storage/sqlite3/previous_events_table.go b/roomserver/storage/sqlite3/previous_events_table.go index 7304bf0d5..2a146ef64 100644 --- a/roomserver/storage/sqlite3/previous_events_table.go +++ b/roomserver/storage/sqlite3/previous_events_table.go @@ -70,12 +70,12 @@ type previousEventStatements struct { selectPreviousEventExistsStmt *sql.Stmt } -func createPrevEventsTable(db *sql.DB) error { +func CreatePrevEventsTable(db *sql.DB) error { _, err := db.Exec(previousEventSchema) return err } -func preparePrevEventsTable(db *sql.DB) (tables.PreviousEvents, error) { +func PreparePrevEventsTable(db *sql.DB) (tables.PreviousEvents, error) { s := &previousEventStatements{ db: db, } diff --git a/roomserver/storage/sqlite3/published_table.go b/roomserver/storage/sqlite3/published_table.go index 9e416ace3..50dfa5492 100644 --- a/roomserver/storage/sqlite3/published_table.go +++ b/roomserver/storage/sqlite3/published_table.go @@ -49,12 +49,12 @@ type publishedStatements struct { selectPublishedStmt *sql.Stmt } -func createPublishedTable(db *sql.DB) error { +func CreatePublishedTable(db *sql.DB) error { _, err := db.Exec(publishedSchema) return err } -func preparePublishedTable(db *sql.DB) (tables.Published, error) { +func PreparePublishedTable(db *sql.DB) (tables.Published, error) { s := &publishedStatements{ db: db, } @@ -96,8 +96,8 @@ func (s *publishedStatements) SelectAllPublishedRooms( defer internal.CloseAndLogIfError(ctx, rows, "selectAllPublishedStmt: rows.close() failed") var roomIDs []string + var roomID string for rows.Next() { - var roomID string if err = rows.Scan(&roomID); err != nil { return nil, err } diff --git a/roomserver/storage/sqlite3/redactions_table.go b/roomserver/storage/sqlite3/redactions_table.go index aed190b1e..db6f57a1b 100644 --- a/roomserver/storage/sqlite3/redactions_table.go +++ b/roomserver/storage/sqlite3/redactions_table.go @@ -48,7 +48,7 @@ const selectRedactionInfoByEventBeingRedactedSQL = "" + " WHERE redacts_event_id = $1" const markRedactionValidatedSQL = "" + - " UPDATE roomserver_redactions SET validated = $2 WHERE redaction_event_id = $1" + " UPDATE roomserver_redactions SET validated = $1 WHERE redaction_event_id = $2" type redactionStatements struct { db *sql.DB @@ -58,12 +58,12 @@ type redactionStatements struct { markRedactionValidatedStmt *sql.Stmt } -func createRedactionsTable(db *sql.DB) error { +func CreateRedactionsTable(db *sql.DB) error { _, err := db.Exec(redactionsSchema) return err } -func prepareRedactionsTable(db *sql.DB) (tables.Redactions, error) { +func PrepareRedactionsTable(db *sql.DB) (tables.Redactions, error) { s := &redactionStatements{ db: db, } @@ -118,6 +118,6 @@ func (s *redactionStatements) MarkRedactionValidated( ctx context.Context, txn *sql.Tx, redactionEventID string, validated bool, ) error { stmt := sqlutil.TxStmt(txn, s.markRedactionValidatedStmt) - _, err := stmt.ExecContext(ctx, redactionEventID, validated) + _, err := stmt.ExecContext(ctx, validated, redactionEventID) return err } diff --git a/roomserver/storage/sqlite3/room_aliases_table.go b/roomserver/storage/sqlite3/room_aliases_table.go index 7c7bead95..3bdbbaa35 100644 --- a/roomserver/storage/sqlite3/room_aliases_table.go +++ b/roomserver/storage/sqlite3/room_aliases_table.go @@ -63,12 +63,12 @@ type roomAliasesStatements struct { deleteRoomAliasStmt *sql.Stmt } -func createRoomAliasesTable(db *sql.DB) error { +func CreateRoomAliasesTable(db *sql.DB) error { _, err := db.Exec(roomAliasesSchema) return err } -func prepareRoomAliasesTable(db *sql.DB) (tables.RoomAliases, error) { +func PrepareRoomAliasesTable(db *sql.DB) (tables.RoomAliases, error) { s := &roomAliasesStatements{ db: db, } @@ -113,8 +113,8 @@ func (s *roomAliasesStatements) SelectAliasesFromRoomID( defer internal.CloseAndLogIfError(ctx, rows, "selectAliasesFromRoomID: rows.close() failed") + var alias string for rows.Next() { - var alias string if err = rows.Scan(&alias); err != nil { return } diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go index cd60c6785..03ad4b3d0 100644 --- a/roomserver/storage/sqlite3/rooms_table.go +++ b/roomserver/storage/sqlite3/rooms_table.go @@ -86,12 +86,12 @@ type roomStatements struct { selectRoomIDsStmt *sql.Stmt } -func createRoomsTable(db *sql.DB) error { +func CreateRoomsTable(db *sql.DB) error { _, err := db.Exec(roomsSchema) return err } -func prepareRoomsTable(db *sql.DB) (tables.Rooms, error) { +func PrepareRoomsTable(db *sql.DB) (tables.Rooms, error) { s := &roomStatements{ db: db, } @@ -108,7 +108,7 @@ func prepareRoomsTable(db *sql.DB) (tables.Rooms, error) { }.Prepare(db) } -func (s *roomStatements) SelectRoomIDs(ctx context.Context, txn *sql.Tx) ([]string, error) { +func (s *roomStatements) SelectRoomIDsWithEvents(ctx context.Context, txn *sql.Tx) ([]string, error) { stmt := sqlutil.TxStmt(txn, s.selectRoomIDsStmt) rows, err := stmt.QueryContext(ctx) if err != nil { @@ -116,8 +116,8 @@ func (s *roomStatements) SelectRoomIDs(ctx context.Context, txn *sql.Tx) ([]stri } defer internal.CloseAndLogIfError(ctx, rows, "selectRoomIDsStmt: rows.close() failed") var roomIDs []string + var roomID string for rows.Next() { - var roomID string if err = rows.Scan(&roomID); err != nil { return nil, err } @@ -241,9 +241,9 @@ func (s *roomStatements) SelectRoomVersionsForRoomNIDs( } defer internal.CloseAndLogIfError(ctx, rows, "selectRoomVersionsForRoomNIDsStmt: rows.close() failed") result := make(map[types.RoomNID]gomatrixserverlib.RoomVersion) + var roomNID types.RoomNID + var roomVersion gomatrixserverlib.RoomVersion for rows.Next() { - var roomNID types.RoomNID - var roomVersion gomatrixserverlib.RoomVersion if err = rows.Scan(&roomNID, &roomVersion); err != nil { return nil, err } @@ -270,8 +270,8 @@ func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roo } defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectRoomIDsStmt: rows.close() failed") var roomIDs []string + var roomID string for rows.Next() { - var roomID string if err = rows.Scan(&roomID); err != nil { return nil, err } @@ -298,8 +298,8 @@ func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, ro } defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectRoomNIDsStmt: rows.close() failed") var roomNIDs []types.RoomNID + var roomNID types.RoomNID for rows.Next() { - var roomNID types.RoomNID if err = rows.Scan(&roomNID); err != nil { return nil, err } diff --git a/roomserver/storage/sqlite3/state_block_table.go b/roomserver/storage/sqlite3/state_block_table.go index 3c829cdcd..4e67d4da1 100644 --- a/roomserver/storage/sqlite3/state_block_table.go +++ b/roomserver/storage/sqlite3/state_block_table.go @@ -20,7 +20,6 @@ import ( "database/sql" "encoding/json" "fmt" - "sort" "strings" "github.com/matrix-org/dendrite/internal" @@ -64,12 +63,12 @@ type stateBlockStatements struct { bulkSelectStateBlockEntriesStmt *sql.Stmt } -func createStateBlockTable(db *sql.DB) error { +func CreateStateBlockTable(db *sql.DB) error { _, err := db.Exec(stateDataSchema) return err } -func prepareStateBlockTable(db *sql.DB) (tables.StateBlock, error) { +func PrepareStateBlockTable(db *sql.DB) (tables.StateBlock, error) { s := &stateBlockStatements{ db: db, } @@ -85,9 +84,9 @@ func (s *stateBlockStatements) BulkInsertStateData( entries types.StateEntries, ) (id types.StateBlockNID, err error) { entries = entries[:util.SortAndUnique(entries)] - nids := types.EventNIDs{} // zero slice to not store 'null' in the DB - for _, e := range entries { - nids = append(nids, e.EventNID) + nids := make(types.EventNIDs, entries.Len()) + for i := range entries { + nids[i] = entries[i].EventNID } js, err := json.Marshal(nids) if err != nil { @@ -122,13 +121,13 @@ func (s *stateBlockStatements) BulkSelectStateBlockEntries( results := make([][]types.EventNID, len(stateBlockNIDs)) i := 0 + var stateBlockNID types.StateBlockNID + var result json.RawMessage for ; rows.Next(); i++ { - var stateBlockNID types.StateBlockNID - var result json.RawMessage if err = rows.Scan(&stateBlockNID, &result); err != nil { return nil, err } - r := []types.EventNID{} + var r []types.EventNID if err = json.Unmarshal(result, &r); err != nil { return nil, fmt.Errorf("json.Unmarshal: %w", err) } @@ -142,35 +141,3 @@ func (s *stateBlockStatements) BulkSelectStateBlockEntries( } return results, err } - -type stateKeyTupleSorter []types.StateKeyTuple - -func (s stateKeyTupleSorter) Len() int { return len(s) } -func (s stateKeyTupleSorter) Less(i, j int) bool { return s[i].LessThan(s[j]) } -func (s stateKeyTupleSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -// Check whether a tuple is in the list. Assumes that the list is sorted. -func (s stateKeyTupleSorter) contains(value types.StateKeyTuple) bool { - i := sort.Search(len(s), func(i int) bool { return !s[i].LessThan(value) }) - return i < len(s) && s[i] == value -} - -// List the unique eventTypeNIDs and eventStateKeyNIDs. -// Assumes that the list is sorted. -func (s stateKeyTupleSorter) typesAndStateKeysAsArrays() (eventTypeNIDs []int64, eventStateKeyNIDs []int64) { - eventTypeNIDs = make([]int64, len(s)) - eventStateKeyNIDs = make([]int64, len(s)) - for i := range s { - eventTypeNIDs[i] = int64(s[i].EventTypeNID) - eventStateKeyNIDs[i] = int64(s[i].EventStateKeyNID) - } - eventTypeNIDs = eventTypeNIDs[:util.SortAndUnique(int64Sorter(eventTypeNIDs))] - eventStateKeyNIDs = eventStateKeyNIDs[:util.SortAndUnique(int64Sorter(eventStateKeyNIDs))] - return -} - -type int64Sorter []int64 - -func (s int64Sorter) Len() int { return len(s) } -func (s int64Sorter) Less(i, j int) bool { return s[i] < s[j] } -func (s int64Sorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } diff --git a/roomserver/storage/sqlite3/state_block_table_test.go b/roomserver/storage/sqlite3/state_block_table_test.go deleted file mode 100644 index 98439f5c0..000000000 --- a/roomserver/storage/sqlite3/state_block_table_test.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sqlite3 - -import ( - "sort" - "testing" - - "github.com/matrix-org/dendrite/roomserver/types" -) - -func TestStateKeyTupleSorter(t *testing.T) { - input := stateKeyTupleSorter{ - {EventTypeNID: 1, EventStateKeyNID: 2}, - {EventTypeNID: 1, EventStateKeyNID: 4}, - {EventTypeNID: 2, EventStateKeyNID: 2}, - {EventTypeNID: 1, EventStateKeyNID: 1}, - } - want := []types.StateKeyTuple{ - {EventTypeNID: 1, EventStateKeyNID: 1}, - {EventTypeNID: 1, EventStateKeyNID: 2}, - {EventTypeNID: 1, EventStateKeyNID: 4}, - {EventTypeNID: 2, EventStateKeyNID: 2}, - } - doNotWant := []types.StateKeyTuple{ - {EventTypeNID: 0, EventStateKeyNID: 0}, - {EventTypeNID: 1, EventStateKeyNID: 3}, - {EventTypeNID: 2, EventStateKeyNID: 1}, - {EventTypeNID: 3, EventStateKeyNID: 1}, - } - wantTypeNIDs := []int64{1, 2} - wantStateKeyNIDs := []int64{1, 2, 4} - - // Sort the input and check it's in the right order. - sort.Sort(input) - gotTypeNIDs, gotStateKeyNIDs := input.typesAndStateKeysAsArrays() - - for i := range want { - if input[i] != want[i] { - t.Errorf("Wanted %#v at index %d got %#v", want[i], i, input[i]) - } - - if !input.contains(want[i]) { - t.Errorf("Wanted %#v.contains(%#v) to be true but got false", input, want[i]) - } - } - - for i := range doNotWant { - if input.contains(doNotWant[i]) { - t.Errorf("Wanted %#v.contains(%#v) to be false but got true", input, doNotWant[i]) - } - } - - if len(wantTypeNIDs) != len(gotTypeNIDs) { - t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs) - } - - for i := range wantTypeNIDs { - if wantTypeNIDs[i] != gotTypeNIDs[i] { - t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs) - } - } - - if len(wantStateKeyNIDs) != len(gotStateKeyNIDs) { - t.Fatalf("Wanted state key NIDs %#v got %#v", wantStateKeyNIDs, gotStateKeyNIDs) - } - - for i := range wantStateKeyNIDs { - if wantStateKeyNIDs[i] != gotStateKeyNIDs[i] { - t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs) - } - } -} diff --git a/roomserver/storage/sqlite3/state_snapshot_table.go b/roomserver/storage/sqlite3/state_snapshot_table.go index 1f5e9ee3b..b8136b758 100644 --- a/roomserver/storage/sqlite3/state_snapshot_table.go +++ b/roomserver/storage/sqlite3/state_snapshot_table.go @@ -68,12 +68,12 @@ type stateSnapshotStatements struct { bulkSelectStateBlockNIDsStmt *sql.Stmt } -func createStateSnapshotTable(db *sql.DB) error { +func CreateStateSnapshotTable(db *sql.DB) error { _, err := db.Exec(stateSnapshotSchema) return err } -func prepareStateSnapshotTable(db *sql.DB) (tables.StateSnapshot, error) { +func PrepareStateSnapshotTable(db *sql.DB) (tables.StateSnapshot, error) { s := &stateSnapshotStatements{ db: db, } @@ -96,12 +96,10 @@ func (s *stateSnapshotStatements) InsertState( return } insertStmt := sqlutil.TxStmt(txn, s.insertStateStmt) - var id int64 - err = insertStmt.QueryRowContext(ctx, stateBlockNIDs.Hash(), int64(roomNID), string(stateBlockNIDsJSON)).Scan(&id) + err = insertStmt.QueryRowContext(ctx, stateBlockNIDs.Hash(), int64(roomNID), string(stateBlockNIDsJSON)).Scan(&stateNID) if err != nil { return 0, err } - stateNID = types.StateSnapshotNID(id) return } @@ -127,9 +125,9 @@ func (s *stateSnapshotStatements) BulkSelectStateBlockNIDs( defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectStateBlockNIDs: rows.close() failed") results := make([]types.StateBlockNIDList, len(stateNIDs)) i := 0 + var stateBlockNIDsJSON string for ; rows.Next(); i++ { result := &results[i] - var stateBlockNIDsJSON string if err := rows.Scan(&result.StateSnapshotNID, &stateBlockNIDsJSON); err != nil { return nil, err } diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index 9522d3058..8325fdad5 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -89,31 +89,31 @@ func (d *Database) create(db *sql.DB) error { if err := CreateEventsTable(db); err != nil { return err } - if err := createRoomsTable(db); err != nil { + if err := CreateRoomsTable(db); err != nil { return err } - if err := createStateBlockTable(db); err != nil { + if err := CreateStateBlockTable(db); err != nil { return err } - if err := createStateSnapshotTable(db); err != nil { + if err := CreateStateSnapshotTable(db); err != nil { return err } - if err := createPrevEventsTable(db); err != nil { + if err := CreatePrevEventsTable(db); err != nil { return err } - if err := createRoomAliasesTable(db); err != nil { + if err := CreateRoomAliasesTable(db); err != nil { return err } - if err := createInvitesTable(db); err != nil { + if err := CreateInvitesTable(db); err != nil { return err } - if err := createMembershipTable(db); err != nil { + if err := CreateMembershipTable(db); err != nil { return err } - if err := createPublishedTable(db); err != nil { + if err := CreatePublishedTable(db); err != nil { return err } - if err := createRedactionsTable(db); err != nil { + if err := CreateRedactionsTable(db); err != nil { return err } @@ -137,39 +137,39 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room if err != nil { return err } - rooms, err := prepareRoomsTable(db) + rooms, err := PrepareRoomsTable(db) if err != nil { return err } - stateBlock, err := prepareStateBlockTable(db) + stateBlock, err := PrepareStateBlockTable(db) if err != nil { return err } - stateSnapshot, err := prepareStateSnapshotTable(db) + stateSnapshot, err := PrepareStateSnapshotTable(db) if err != nil { return err } - prevEvents, err := preparePrevEventsTable(db) + prevEvents, err := PreparePrevEventsTable(db) if err != nil { return err } - roomAliases, err := prepareRoomAliasesTable(db) + roomAliases, err := PrepareRoomAliasesTable(db) if err != nil { return err } - invites, err := prepareInvitesTable(db) + invites, err := PrepareInvitesTable(db) if err != nil { return err } - membership, err := prepareMembershipTable(db) + membership, err := PrepareMembershipTable(db) if err != nil { return err } - published, err := preparePublishedTable(db) + published, err := PreparePublishedTable(db) if err != nil { return err } - redactions, err := prepareRedactionsTable(db) + redactions, err := PrepareRedactionsTable(db) if err != nil { return err } diff --git a/roomserver/storage/tables/events_table_test.go b/roomserver/storage/tables/events_table_test.go index d5d699c4c..6f72a59b5 100644 --- a/roomserver/storage/tables/events_table_test.go +++ b/roomserver/storage/tables/events_table_test.go @@ -39,7 +39,7 @@ func mustCreateEventsTable(t *testing.T, dbType test.DBType) (tables.Events, fun } func Test_EventsTable(t *testing.T) { - alice := test.NewUser() + alice := test.NewUser(t) room := test.NewRoom(t, alice) ctx := context.Background() test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go index 95609787a..116e11c4e 100644 --- a/roomserver/storage/tables/interface.go +++ b/roomserver/storage/tables/interface.go @@ -72,7 +72,7 @@ type Rooms interface { UpdateLatestEventNIDs(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, eventNIDs []types.EventNID, lastEventSentNID types.EventNID, stateSnapshotNID types.StateSnapshotNID) error SelectRoomVersionsForRoomNIDs(ctx context.Context, txn *sql.Tx, roomNID []types.RoomNID) (map[types.RoomNID]gomatrixserverlib.RoomVersion, error) SelectRoomInfo(ctx context.Context, txn *sql.Tx, roomID string) (*types.RoomInfo, error) - SelectRoomIDs(ctx context.Context, txn *sql.Tx) ([]string, error) + SelectRoomIDsWithEvents(ctx context.Context, txn *sql.Tx) ([]string, error) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID) ([]string, error) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, roomIDs []string) ([]types.RoomNID, error) } diff --git a/roomserver/storage/tables/invite_table_test.go b/roomserver/storage/tables/invite_table_test.go new file mode 100644 index 000000000..8df3faa2d --- /dev/null +++ b/roomserver/storage/tables/invite_table_test.go @@ -0,0 +1,92 @@ +package tables_test + +import ( + "context" + "testing" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver/storage/postgres" + "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" + "github.com/matrix-org/dendrite/roomserver/storage/tables" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/util" + "github.com/stretchr/testify/assert" +) + +func mustCreateInviteTable(t *testing.T, dbType test.DBType) (tables.Invites, func()) { + t.Helper() + connStr, close := test.PrepareDBConnectionString(t, dbType) + db, err := sqlutil.Open(&config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + }, sqlutil.NewExclusiveWriter()) + assert.NoError(t, err) + var tab tables.Invites + switch dbType { + case test.DBTypePostgres: + err = postgres.CreateInvitesTable(db) + assert.NoError(t, err) + tab, err = postgres.PrepareInvitesTable(db) + case test.DBTypeSQLite: + err = sqlite3.CreateInvitesTable(db) + assert.NoError(t, err) + tab, err = sqlite3.PrepareInvitesTable(db) + } + assert.NoError(t, err) + + return tab, close +} + +func TestInviteTable(t *testing.T) { + ctx := context.Background() + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + tab, close := mustCreateInviteTable(t, dbType) + defer close() + eventID1 := util.RandomString(16) + roomNID := types.RoomNID(1) + targetUserNID, senderUserNID := types.EventStateKeyNID(1), types.EventStateKeyNID(2) + newInvite, err := tab.InsertInviteEvent(ctx, nil, eventID1, roomNID, targetUserNID, senderUserNID, []byte("")) + assert.NoError(t, err) + assert.True(t, newInvite) + + // Try adding the same invite again + newInvite, err = tab.InsertInviteEvent(ctx, nil, eventID1, roomNID, targetUserNID, senderUserNID, []byte("")) + assert.NoError(t, err) + assert.False(t, newInvite) + + // Add another invite for this room + eventID2 := util.RandomString(16) + newInvite, err = tab.InsertInviteEvent(ctx, nil, eventID2, roomNID, targetUserNID, senderUserNID, []byte("")) + assert.NoError(t, err) + assert.True(t, newInvite) + + // Add another invite for a different user + eventID := util.RandomString(16) + newInvite, err = tab.InsertInviteEvent(ctx, nil, eventID, types.RoomNID(3), targetUserNID, senderUserNID, []byte("")) + assert.NoError(t, err) + assert.True(t, newInvite) + + stateKeyNIDs, eventIDs, err := tab.SelectInviteActiveForUserInRoom(ctx, nil, targetUserNID, roomNID) + assert.NoError(t, err) + assert.Equal(t, []string{eventID1, eventID2}, eventIDs) + assert.Equal(t, []types.EventStateKeyNID{2, 2}, stateKeyNIDs) + + // retire the invite + retiredEventIDs, err := tab.UpdateInviteRetired(ctx, nil, roomNID, targetUserNID) + assert.NoError(t, err) + assert.Equal(t, []string{eventID1, eventID2}, retiredEventIDs) + + // This should now be empty + stateKeyNIDs, eventIDs, err = tab.SelectInviteActiveForUserInRoom(ctx, nil, targetUserNID, roomNID) + assert.NoError(t, err) + assert.Empty(t, eventIDs) + assert.Empty(t, stateKeyNIDs) + + // Non-existent targetUserNID + stateKeyNIDs, eventIDs, err = tab.SelectInviteActiveForUserInRoom(ctx, nil, types.EventStateKeyNID(10), roomNID) + assert.NoError(t, err) + assert.Empty(t, stateKeyNIDs) + assert.Empty(t, eventIDs) + }) +} diff --git a/roomserver/storage/tables/membership_table_test.go b/roomserver/storage/tables/membership_table_test.go new file mode 100644 index 000000000..14e8ce50a --- /dev/null +++ b/roomserver/storage/tables/membership_table_test.go @@ -0,0 +1,130 @@ +package tables_test + +import ( + "context" + "fmt" + "testing" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver/storage/postgres" + "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" + "github.com/matrix-org/dendrite/roomserver/storage/tables" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/test" + "github.com/stretchr/testify/assert" +) + +func mustCreateMembershipTable(t *testing.T, dbType test.DBType) (tab tables.Membership, stateKeyTab tables.EventStateKeys, close func()) { + t.Helper() + connStr, close := test.PrepareDBConnectionString(t, dbType) + db, err := sqlutil.Open(&config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + }, sqlutil.NewExclusiveWriter()) + assert.NoError(t, err) + switch dbType { + case test.DBTypePostgres: + err = postgres.CreateEventStateKeysTable(db) + assert.NoError(t, err) + err = postgres.CreateMembershipTable(db) + assert.NoError(t, err) + tab, err = postgres.PrepareMembershipTable(db) + assert.NoError(t, err) + stateKeyTab, err = postgres.PrepareEventStateKeysTable(db) + case test.DBTypeSQLite: + err = sqlite3.CreateEventStateKeysTable(db) + assert.NoError(t, err) + err = sqlite3.CreateMembershipTable(db) + assert.NoError(t, err) + tab, err = sqlite3.PrepareMembershipTable(db) + assert.NoError(t, err) + stateKeyTab, err = sqlite3.PrepareEventStateKeysTable(db) + } + assert.NoError(t, err) + + return tab, stateKeyTab, close +} + +func TestMembershipTable(t *testing.T) { + ctx := context.Background() + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + tab, stateKeyTab, close := mustCreateMembershipTable(t, dbType) + defer close() + _ = close + + userNIDs := make([]types.EventStateKeyNID, 0, 10) + for i := 0; i < 10; i++ { + stateKeyNID, err := stateKeyTab.InsertEventStateKeyNID(ctx, nil, fmt.Sprintf("@dummy%d:localhost", i)) + assert.NoError(t, err) + userNIDs = append(userNIDs, stateKeyNID) + // This inserts a left user to the room + err = tab.InsertMembership(ctx, nil, 1, stateKeyNID, true) + assert.NoError(t, err) + } + + // ... so this should be false + inRoom, err := tab.SelectLocalServerInRoom(ctx, nil, 1) + assert.NoError(t, err) + assert.False(t, inRoom) + + changed, err := tab.UpdateMembership(ctx, nil, 1, userNIDs[0], userNIDs[0], tables.MembershipStateJoin, 1, false) + assert.NoError(t, err) + assert.True(t, changed) + + // ... should now be true + inRoom, err = tab.SelectLocalServerInRoom(ctx, nil, 1) + assert.NoError(t, err) + assert.True(t, inRoom) + + userJoinedToRooms, err := tab.SelectJoinedUsersSetForRooms(ctx, nil, []types.RoomNID{1}, userNIDs) + assert.NoError(t, err) + assert.Equal(t, 1, len(userJoinedToRooms)) + + // Get all left/banned users + eventNIDs, err := tab.SelectMembershipsFromRoomAndMembership(ctx, nil, 1, tables.MembershipStateLeaveOrBan, true) + assert.NoError(t, err) + assert.Equal(t, 9, len(eventNIDs)) + + _, membershipState, forgotten, err := tab.SelectMembershipFromRoomAndTarget(ctx, nil, 1, userNIDs[5]) + assert.NoError(t, err) + assert.False(t, forgotten) + assert.Equal(t, tables.MembershipStateLeaveOrBan, membershipState) + + // Get all members, regardless of state + members, err := tab.SelectMembershipsFromRoom(ctx, nil, 1, true) + assert.NoError(t, err) + assert.Equal(t, 10, len(members)) + + // Get correct user + roomNIDs, err := tab.SelectRoomsWithMembership(ctx, nil, userNIDs[1], tables.MembershipStateLeaveOrBan) + assert.NoError(t, err) + assert.Equal(t, []types.RoomNID{1}, roomNIDs) + + // User is not joined to room + roomNIDs, err = tab.SelectRoomsWithMembership(ctx, nil, userNIDs[5], tables.MembershipStateJoin) + assert.NoError(t, err) + assert.Equal(t, 0, len(roomNIDs)) + + // Forget room + err = tab.UpdateForgetMembership(ctx, nil, 1, userNIDs[0], true) + assert.NoError(t, err) + + // should now return true + _, _, forgotten, err = tab.SelectMembershipFromRoomAndTarget(ctx, nil, 1, userNIDs[0]) + assert.NoError(t, err) + assert.True(t, forgotten) + + serverInRoom, err := tab.SelectServerInRoom(ctx, nil, 1, "localhost") + assert.NoError(t, err) + assert.True(t, serverInRoom) + + serverInRoom, err = tab.SelectServerInRoom(ctx, nil, 1, "notJoined") + assert.NoError(t, err) + assert.False(t, serverInRoom) + + // get all users we know about; should be only one user, since no other user joined the room + knownUsers, err := tab.SelectKnownUsers(ctx, nil, userNIDs[0], "localhost", 2) + assert.NoError(t, err) + assert.Equal(t, 1, len(knownUsers)) + }) +} diff --git a/roomserver/storage/tables/previous_events_table_test.go b/roomserver/storage/tables/previous_events_table_test.go new file mode 100644 index 000000000..63d540696 --- /dev/null +++ b/roomserver/storage/tables/previous_events_table_test.go @@ -0,0 +1,61 @@ +package tables_test + +import ( + "context" + "testing" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver/storage/postgres" + "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" + "github.com/matrix-org/dendrite/roomserver/storage/tables" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/util" + "github.com/stretchr/testify/assert" +) + +func mustCreatePreviousEventsTable(t *testing.T, dbType test.DBType) (tab tables.PreviousEvents, close func()) { + t.Helper() + connStr, close := test.PrepareDBConnectionString(t, dbType) + db, err := sqlutil.Open(&config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + }, sqlutil.NewExclusiveWriter()) + assert.NoError(t, err) + switch dbType { + case test.DBTypePostgres: + err = postgres.CreatePrevEventsTable(db) + assert.NoError(t, err) + tab, err = postgres.PreparePrevEventsTable(db) + case test.DBTypeSQLite: + err = sqlite3.CreatePrevEventsTable(db) + assert.NoError(t, err) + tab, err = sqlite3.PreparePrevEventsTable(db) + } + assert.NoError(t, err) + + return tab, close +} + +func TestPreviousEventsTable(t *testing.T) { + ctx := context.Background() + alice := test.NewUser(t) + room := test.NewRoom(t, alice) + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + tab, close := mustCreatePreviousEventsTable(t, dbType) + defer close() + + for _, x := range room.Events() { + for _, prevEvent := range x.PrevEvents() { + err := tab.InsertPreviousEvent(ctx, nil, prevEvent.EventID, prevEvent.EventSHA256, 1) + assert.NoError(t, err) + + err = tab.SelectPreviousEventExists(ctx, nil, prevEvent.EventID, prevEvent.EventSHA256) + assert.NoError(t, err) + } + } + + // RandomString with a correct EventSHA256 should fail and return sql.ErrNoRows + err := tab.SelectPreviousEventExists(ctx, nil, util.RandomString(16), room.Events()[0].EventReference().EventSHA256) + assert.Error(t, err) + }) +} diff --git a/roomserver/storage/tables/published_table_test.go b/roomserver/storage/tables/published_table_test.go new file mode 100644 index 000000000..fff6dc186 --- /dev/null +++ b/roomserver/storage/tables/published_table_test.go @@ -0,0 +1,79 @@ +package tables_test + +import ( + "context" + "sort" + "testing" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver/storage/postgres" + "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" + "github.com/matrix-org/dendrite/roomserver/storage/tables" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/test" + "github.com/stretchr/testify/assert" +) + +func mustCreatePublishedTable(t *testing.T, dbType test.DBType) (tab tables.Published, close func()) { + t.Helper() + connStr, close := test.PrepareDBConnectionString(t, dbType) + db, err := sqlutil.Open(&config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + }, sqlutil.NewExclusiveWriter()) + assert.NoError(t, err) + switch dbType { + case test.DBTypePostgres: + err = postgres.CreatePublishedTable(db) + assert.NoError(t, err) + tab, err = postgres.PreparePublishedTable(db) + case test.DBTypeSQLite: + err = sqlite3.CreatePublishedTable(db) + assert.NoError(t, err) + tab, err = sqlite3.PreparePublishedTable(db) + } + assert.NoError(t, err) + + return tab, close +} + +func TestPublishedTable(t *testing.T) { + ctx := context.Background() + alice := test.NewUser(t) + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + tab, close := mustCreatePublishedTable(t, dbType) + defer close() + + // Publish some rooms + publishedRooms := []string{} + for i := 0; i < 10; i++ { + room := test.NewRoom(t, alice) + published := i%2 == 0 + err := tab.UpsertRoomPublished(ctx, nil, room.ID, published) + assert.NoError(t, err) + if published { + publishedRooms = append(publishedRooms, room.ID) + } + publishedRes, err := tab.SelectPublishedFromRoomID(ctx, nil, room.ID) + assert.NoError(t, err) + assert.Equal(t, published, publishedRes) + } + sort.Strings(publishedRooms) + + // check that we get the expected published rooms + roomIDs, err := tab.SelectAllPublishedRooms(ctx, nil, true) + assert.NoError(t, err) + assert.Equal(t, publishedRooms, roomIDs) + + // test an actual upsert + room := test.NewRoom(t, alice) + err = tab.UpsertRoomPublished(ctx, nil, room.ID, true) + assert.NoError(t, err) + err = tab.UpsertRoomPublished(ctx, nil, room.ID, false) + assert.NoError(t, err) + // should now be false, due to the upsert + publishedRes, err := tab.SelectPublishedFromRoomID(ctx, nil, room.ID) + assert.NoError(t, err) + assert.False(t, publishedRes) + }) +} diff --git a/roomserver/storage/tables/redactions_table_test.go b/roomserver/storage/tables/redactions_table_test.go new file mode 100644 index 000000000..ea48dc22f --- /dev/null +++ b/roomserver/storage/tables/redactions_table_test.go @@ -0,0 +1,89 @@ +package tables_test + +import ( + "context" + "testing" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver/storage/postgres" + "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" + "github.com/matrix-org/dendrite/roomserver/storage/tables" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/util" + "github.com/stretchr/testify/assert" +) + +func mustCreateRedactionsTable(t *testing.T, dbType test.DBType) (tab tables.Redactions, close func()) { + t.Helper() + connStr, close := test.PrepareDBConnectionString(t, dbType) + db, err := sqlutil.Open(&config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + }, sqlutil.NewExclusiveWriter()) + assert.NoError(t, err) + switch dbType { + case test.DBTypePostgres: + err = postgres.CreateRedactionsTable(db) + assert.NoError(t, err) + tab, err = postgres.PrepareRedactionsTable(db) + case test.DBTypeSQLite: + err = sqlite3.CreateRedactionsTable(db) + assert.NoError(t, err) + tab, err = sqlite3.PrepareRedactionsTable(db) + } + assert.NoError(t, err) + + return tab, close +} + +func TestRedactionsTable(t *testing.T) { + ctx := context.Background() + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + tab, close := mustCreateRedactionsTable(t, dbType) + defer close() + + // insert and verify some redactions + for i := 0; i < 10; i++ { + redactionEventID, redactsEventID := util.RandomString(16), util.RandomString(16) + wantRedactionInfo := tables.RedactionInfo{ + Validated: false, + RedactsEventID: redactsEventID, + RedactionEventID: redactionEventID, + } + err := tab.InsertRedaction(ctx, nil, wantRedactionInfo) + assert.NoError(t, err) + + // verify the redactions are inserted as expected + redactionInfo, err := tab.SelectRedactionInfoByRedactionEventID(ctx, nil, redactionEventID) + assert.NoError(t, err) + assert.Equal(t, &wantRedactionInfo, redactionInfo) + + redactionInfo, err = tab.SelectRedactionInfoByEventBeingRedacted(ctx, nil, redactsEventID) + assert.NoError(t, err) + assert.Equal(t, &wantRedactionInfo, redactionInfo) + + // redact event + err = tab.MarkRedactionValidated(ctx, nil, redactionEventID, true) + assert.NoError(t, err) + + wantRedactionInfo.Validated = true + redactionInfo, err = tab.SelectRedactionInfoByRedactionEventID(ctx, nil, redactionEventID) + assert.NoError(t, err) + assert.Equal(t, &wantRedactionInfo, redactionInfo) + } + + // Should not fail, it just updates 0 rows + err := tab.MarkRedactionValidated(ctx, nil, "iDontExist", true) + assert.NoError(t, err) + + // Should also not fail, but return a nil redactionInfo + redactionInfo, err := tab.SelectRedactionInfoByRedactionEventID(ctx, nil, "iDontExist") + assert.NoError(t, err) + assert.Nil(t, redactionInfo) + + redactionInfo, err = tab.SelectRedactionInfoByEventBeingRedacted(ctx, nil, "iDontExist") + assert.NoError(t, err) + assert.Nil(t, redactionInfo) + }) +} diff --git a/roomserver/storage/tables/room_aliases_table_test.go b/roomserver/storage/tables/room_aliases_table_test.go new file mode 100644 index 000000000..624d92ae6 --- /dev/null +++ b/roomserver/storage/tables/room_aliases_table_test.go @@ -0,0 +1,96 @@ +package tables_test + +import ( + "context" + "testing" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver/storage/postgres" + "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" + "github.com/matrix-org/dendrite/roomserver/storage/tables" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/test" + "github.com/stretchr/testify/assert" +) + +func mustCreateRoomAliasesTable(t *testing.T, dbType test.DBType) (tab tables.RoomAliases, close func()) { + t.Helper() + connStr, close := test.PrepareDBConnectionString(t, dbType) + db, err := sqlutil.Open(&config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + }, sqlutil.NewExclusiveWriter()) + assert.NoError(t, err) + switch dbType { + case test.DBTypePostgres: + err = postgres.CreateRoomAliasesTable(db) + assert.NoError(t, err) + tab, err = postgres.PrepareRoomAliasesTable(db) + case test.DBTypeSQLite: + err = sqlite3.CreateRoomAliasesTable(db) + assert.NoError(t, err) + tab, err = sqlite3.PrepareRoomAliasesTable(db) + } + assert.NoError(t, err) + + return tab, close +} + +func TestRoomAliasesTable(t *testing.T) { + alice := test.NewUser(t) + room := test.NewRoom(t, alice) + room2 := test.NewRoom(t, alice) + ctx := context.Background() + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + tab, close := mustCreateRoomAliasesTable(t, dbType) + defer close() + alias, alias2, alias3 := "#alias:localhost", "#alias2:localhost", "#alias3:localhost" + // insert aliases + err := tab.InsertRoomAlias(ctx, nil, alias, room.ID, alice.ID) + assert.NoError(t, err) + + err = tab.InsertRoomAlias(ctx, nil, alias2, room.ID, alice.ID) + assert.NoError(t, err) + + err = tab.InsertRoomAlias(ctx, nil, alias3, room2.ID, alice.ID) + assert.NoError(t, err) + + // verify we can get the roomID for the alias + roomID, err := tab.SelectRoomIDFromAlias(ctx, nil, alias) + assert.NoError(t, err) + assert.Equal(t, room.ID, roomID) + + // .. and the creator + creator, err := tab.SelectCreatorIDFromAlias(ctx, nil, alias) + assert.NoError(t, err) + assert.Equal(t, alice.ID, creator) + + creator, err = tab.SelectCreatorIDFromAlias(ctx, nil, "#doesntexist:localhost") + assert.NoError(t, err) + assert.Equal(t, "", creator) + + roomID, err = tab.SelectRoomIDFromAlias(ctx, nil, "#doesntexist:localhost") + assert.NoError(t, err) + assert.Equal(t, "", roomID) + + // get all aliases for a room + aliases, err := tab.SelectAliasesFromRoomID(ctx, nil, room.ID) + assert.NoError(t, err) + assert.Equal(t, []string{alias, alias2}, aliases) + + // delete an alias and verify it's deleted + err = tab.DeleteRoomAlias(ctx, nil, alias2) + assert.NoError(t, err) + + aliases, err = tab.SelectAliasesFromRoomID(ctx, nil, room.ID) + assert.NoError(t, err) + assert.Equal(t, []string{alias}, aliases) + + // deleting the same alias should be a no-op + err = tab.DeleteRoomAlias(ctx, nil, alias2) + assert.NoError(t, err) + + // Delete non-existent alias should be a no-op + err = tab.DeleteRoomAlias(ctx, nil, "#doesntexist:localhost") + assert.NoError(t, err) + }) +} diff --git a/roomserver/storage/tables/rooms_table_test.go b/roomserver/storage/tables/rooms_table_test.go new file mode 100644 index 000000000..0a02369a1 --- /dev/null +++ b/roomserver/storage/tables/rooms_table_test.go @@ -0,0 +1,128 @@ +package tables_test + +import ( + "context" + "testing" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver/storage/postgres" + "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" + "github.com/matrix-org/dendrite/roomserver/storage/tables" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/util" + "github.com/stretchr/testify/assert" +) + +func mustCreateRoomsTable(t *testing.T, dbType test.DBType) (tab tables.Rooms, close func()) { + t.Helper() + connStr, close := test.PrepareDBConnectionString(t, dbType) + db, err := sqlutil.Open(&config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + }, sqlutil.NewExclusiveWriter()) + assert.NoError(t, err) + switch dbType { + case test.DBTypePostgres: + err = postgres.CreateRoomsTable(db) + assert.NoError(t, err) + tab, err = postgres.PrepareRoomsTable(db) + case test.DBTypeSQLite: + err = sqlite3.CreateRoomsTable(db) + assert.NoError(t, err) + tab, err = sqlite3.PrepareRoomsTable(db) + } + assert.NoError(t, err) + + return tab, close +} + +func TestRoomsTable(t *testing.T) { + alice := test.NewUser(t) + room := test.NewRoom(t, alice) + ctx := context.Background() + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + tab, close := mustCreateRoomsTable(t, dbType) + defer close() + + wantRoomNID, err := tab.InsertRoomNID(ctx, nil, room.ID, room.Version) + assert.NoError(t, err) + + // Create dummy room + _, err = tab.InsertRoomNID(ctx, nil, util.RandomString(16), room.Version) + assert.NoError(t, err) + + gotRoomNID, err := tab.SelectRoomNID(ctx, nil, room.ID) + assert.NoError(t, err) + assert.Equal(t, wantRoomNID, gotRoomNID) + + // Ensure non existent roomNID errors + roomNID, err := tab.SelectRoomNID(ctx, nil, "!doesnotexist:localhost") + assert.Error(t, err) + assert.Equal(t, types.RoomNID(0), roomNID) + + roomInfo, err := tab.SelectRoomInfo(ctx, nil, room.ID) + assert.NoError(t, err) + assert.Equal(t, &types.RoomInfo{ + RoomNID: wantRoomNID, + RoomVersion: room.Version, + StateSnapshotNID: 0, + IsStub: true, // there are no latestEventNIDs + }, roomInfo) + + roomInfo, err = tab.SelectRoomInfo(ctx, nil, "!doesnotexist:localhost") + assert.NoError(t, err) + assert.Nil(t, roomInfo) + + // There are no rooms with latestEventNIDs yet + roomIDs, err := tab.SelectRoomIDsWithEvents(ctx, nil) + assert.NoError(t, err) + assert.Equal(t, 0, len(roomIDs)) + + roomVersions, err := tab.SelectRoomVersionsForRoomNIDs(ctx, nil, []types.RoomNID{wantRoomNID, 1337}) + assert.NoError(t, err) + assert.Equal(t, roomVersions[wantRoomNID], room.Version) + // Room does not exist + _, ok := roomVersions[1337] + assert.False(t, ok) + + roomIDs, err = tab.BulkSelectRoomIDs(ctx, nil, []types.RoomNID{wantRoomNID, 1337}) + assert.NoError(t, err) + assert.Equal(t, []string{room.ID}, roomIDs) + + roomNIDs, err := tab.BulkSelectRoomNIDs(ctx, nil, []string{room.ID, "!doesnotexist:localhost"}) + assert.NoError(t, err) + assert.Equal(t, []types.RoomNID{wantRoomNID}, roomNIDs) + + wantEventNIDs := []types.EventNID{1, 2, 3} + lastEventSentNID := types.EventNID(3) + stateSnapshotNID := types.StateSnapshotNID(1) + // make the room "usable" + err = tab.UpdateLatestEventNIDs(ctx, nil, wantRoomNID, wantEventNIDs, lastEventSentNID, stateSnapshotNID) + assert.NoError(t, err) + + roomInfo, err = tab.SelectRoomInfo(ctx, nil, room.ID) + assert.NoError(t, err) + assert.Equal(t, &types.RoomInfo{ + RoomNID: wantRoomNID, + RoomVersion: room.Version, + StateSnapshotNID: 1, + IsStub: false, + }, roomInfo) + + eventNIDs, snapshotNID, err := tab.SelectLatestEventNIDs(ctx, nil, wantRoomNID) + assert.NoError(t, err) + assert.Equal(t, wantEventNIDs, eventNIDs) + assert.Equal(t, types.StateSnapshotNID(1), snapshotNID) + + // Again, doesn't exist + _, _, err = tab.SelectLatestEventNIDs(ctx, nil, 1337) + assert.Error(t, err) + + eventNIDs, eventNID, snapshotNID, err := tab.SelectLatestEventsNIDsForUpdate(ctx, nil, wantRoomNID) + assert.NoError(t, err) + assert.Equal(t, wantEventNIDs, eventNIDs) + assert.Equal(t, types.EventNID(3), eventNID) + assert.Equal(t, types.StateSnapshotNID(1), snapshotNID) + }) +} diff --git a/roomserver/storage/tables/state_block_table_test.go b/roomserver/storage/tables/state_block_table_test.go new file mode 100644 index 000000000..de0b420bc --- /dev/null +++ b/roomserver/storage/tables/state_block_table_test.go @@ -0,0 +1,92 @@ +package tables_test + +import ( + "context" + "testing" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver/storage/postgres" + "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" + "github.com/matrix-org/dendrite/roomserver/storage/tables" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/test" + "github.com/stretchr/testify/assert" +) + +func mustCreateStateBlockTable(t *testing.T, dbType test.DBType) (tab tables.StateBlock, close func()) { + t.Helper() + connStr, close := test.PrepareDBConnectionString(t, dbType) + db, err := sqlutil.Open(&config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + }, sqlutil.NewExclusiveWriter()) + assert.NoError(t, err) + switch dbType { + case test.DBTypePostgres: + err = postgres.CreateStateBlockTable(db) + assert.NoError(t, err) + tab, err = postgres.PrepareStateBlockTable(db) + case test.DBTypeSQLite: + err = sqlite3.CreateStateBlockTable(db) + assert.NoError(t, err) + tab, err = sqlite3.PrepareStateBlockTable(db) + } + assert.NoError(t, err) + + return tab, close +} + +func TestStateBlockTable(t *testing.T) { + ctx := context.Background() + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + tab, close := mustCreateStateBlockTable(t, dbType) + defer close() + + // generate some dummy data + var entries types.StateEntries + for i := 0; i < 100; i++ { + entry := types.StateEntry{ + EventNID: types.EventNID(i), + } + entries = append(entries, entry) + } + stateBlockNID, err := tab.BulkInsertStateData(ctx, nil, entries) + assert.NoError(t, err) + assert.Equal(t, types.StateBlockNID(1), stateBlockNID) + + // generate a different hash, to get a new StateBlockNID + var entries2 types.StateEntries + for i := 100; i < 300; i++ { + entry := types.StateEntry{ + EventNID: types.EventNID(i), + } + entries2 = append(entries2, entry) + } + stateBlockNID, err = tab.BulkInsertStateData(ctx, nil, entries2) + assert.NoError(t, err) + assert.Equal(t, types.StateBlockNID(2), stateBlockNID) + + eventNIDs, err := tab.BulkSelectStateBlockEntries(ctx, nil, types.StateBlockNIDs{1, 2}) + assert.NoError(t, err) + assert.Equal(t, len(entries), len(eventNIDs[0])) + assert.Equal(t, len(entries2), len(eventNIDs[1])) + + // try to get a StateBlockNID which does not exist + _, err = tab.BulkSelectStateBlockEntries(ctx, nil, types.StateBlockNIDs{5}) + assert.Error(t, err) + + // This should return an error, since we can only retrieve 1 StateBlock + _, err = tab.BulkSelectStateBlockEntries(ctx, nil, types.StateBlockNIDs{1, 5}) + assert.Error(t, err) + + for i := 0; i < 65555; i++ { + entry := types.StateEntry{ + EventNID: types.EventNID(i), + } + entries2 = append(entries2, entry) + } + stateBlockNID, err = tab.BulkInsertStateData(ctx, nil, entries2) + assert.NoError(t, err) + assert.Equal(t, types.StateBlockNID(3), stateBlockNID) + }) +} diff --git a/roomserver/storage/tables/state_snapshot_table_test.go b/roomserver/storage/tables/state_snapshot_table_test.go new file mode 100644 index 000000000..dcdb5d8f1 --- /dev/null +++ b/roomserver/storage/tables/state_snapshot_table_test.go @@ -0,0 +1,86 @@ +package tables_test + +import ( + "context" + "testing" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver/storage/postgres" + "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" + "github.com/matrix-org/dendrite/roomserver/storage/tables" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/test" + "github.com/stretchr/testify/assert" +) + +func mustCreateStateSnapshotTable(t *testing.T, dbType test.DBType) (tab tables.StateSnapshot, close func()) { + t.Helper() + connStr, close := test.PrepareDBConnectionString(t, dbType) + db, err := sqlutil.Open(&config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + }, sqlutil.NewExclusiveWriter()) + assert.NoError(t, err) + switch dbType { + case test.DBTypePostgres: + err = postgres.CreateStateSnapshotTable(db) + assert.NoError(t, err) + tab, err = postgres.PrepareStateSnapshotTable(db) + case test.DBTypeSQLite: + err = sqlite3.CreateStateSnapshotTable(db) + assert.NoError(t, err) + tab, err = sqlite3.PrepareStateSnapshotTable(db) + } + assert.NoError(t, err) + + return tab, close +} + +func TestStateSnapshotTable(t *testing.T) { + ctx := context.Background() + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + tab, close := mustCreateStateSnapshotTable(t, dbType) + defer close() + + // generate some dummy data + var stateBlockNIDs types.StateBlockNIDs + for i := 0; i < 100; i++ { + stateBlockNIDs = append(stateBlockNIDs, types.StateBlockNID(i)) + } + stateNID, err := tab.InsertState(ctx, nil, 1, stateBlockNIDs) + assert.NoError(t, err) + assert.Equal(t, types.StateSnapshotNID(1), stateNID) + + // verify ON CONFLICT; Note: this updates the sequence! + stateNID, err = tab.InsertState(ctx, nil, 1, stateBlockNIDs) + assert.NoError(t, err) + assert.Equal(t, types.StateSnapshotNID(1), stateNID) + + // create a second snapshot + var stateBlockNIDs2 types.StateBlockNIDs + for i := 100; i < 150; i++ { + stateBlockNIDs2 = append(stateBlockNIDs2, types.StateBlockNID(i)) + } + + stateNID, err = tab.InsertState(ctx, nil, 1, stateBlockNIDs2) + assert.NoError(t, err) + // StateSnapshotNID is now 3, since the DO UPDATE SET statement incremented the sequence + assert.Equal(t, types.StateSnapshotNID(3), stateNID) + + nidLists, err := tab.BulkSelectStateBlockNIDs(ctx, nil, []types.StateSnapshotNID{1, 3}) + assert.NoError(t, err) + assert.Equal(t, stateBlockNIDs, types.StateBlockNIDs(nidLists[0].StateBlockNIDs)) + assert.Equal(t, stateBlockNIDs2, types.StateBlockNIDs(nidLists[1].StateBlockNIDs)) + + // check we get an error if the state snapshot does not exist + _, err = tab.BulkSelectStateBlockNIDs(ctx, nil, []types.StateSnapshotNID{2}) + assert.Error(t, err) + + // create a second snapshot + for i := 0; i < 65555; i++ { + stateBlockNIDs2 = append(stateBlockNIDs2, types.StateBlockNID(i)) + } + _, err = tab.InsertState(ctx, nil, 1, stateBlockNIDs2) + assert.NoError(t, err) + }) +} diff --git a/roomserver/types/types.go b/roomserver/types/types.go index 65fbee04e..62695aaee 100644 --- a/roomserver/types/types.go +++ b/roomserver/types/types.go @@ -18,8 +18,10 @@ package types import ( "encoding/json" "sort" + "strings" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" "golang.org/x/crypto/blake2b" ) @@ -96,6 +98,38 @@ func (a StateKeyTuple) LessThan(b StateKeyTuple) bool { return a.EventStateKeyNID < b.EventStateKeyNID } +type StateKeyTupleSorter []StateKeyTuple + +func (s StateKeyTupleSorter) Len() int { return len(s) } +func (s StateKeyTupleSorter) Less(i, j int) bool { return s[i].LessThan(s[j]) } +func (s StateKeyTupleSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// Check whether a tuple is in the list. Assumes that the list is sorted. +func (s StateKeyTupleSorter) contains(value StateKeyTuple) bool { + i := sort.Search(len(s), func(i int) bool { return !s[i].LessThan(value) }) + return i < len(s) && s[i] == value +} + +// List the unique eventTypeNIDs and eventStateKeyNIDs. +// Assumes that the list is sorted. +func (s StateKeyTupleSorter) TypesAndStateKeysAsArrays() (eventTypeNIDs []int64, eventStateKeyNIDs []int64) { + eventTypeNIDs = make([]int64, len(s)) + eventStateKeyNIDs = make([]int64, len(s)) + for i := range s { + eventTypeNIDs[i] = int64(s[i].EventTypeNID) + eventStateKeyNIDs[i] = int64(s[i].EventStateKeyNID) + } + eventTypeNIDs = eventTypeNIDs[:util.SortAndUnique(int64Sorter(eventTypeNIDs))] + eventStateKeyNIDs = eventStateKeyNIDs[:util.SortAndUnique(int64Sorter(eventStateKeyNIDs))] + return +} + +type int64Sorter []int64 + +func (s int64Sorter) Len() int { return len(s) } +func (s int64Sorter) Less(i, j int) bool { return s[i] < s[j] } +func (s int64Sorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + // A StateEntry is an entry in the room state of a matrix room. type StateEntry struct { StateKeyTuple @@ -166,6 +200,20 @@ type StateAtEventAndReference struct { gomatrixserverlib.EventReference } +type StateAtEventAndReferences []StateAtEventAndReference + +func (s StateAtEventAndReferences) Less(a, b int) bool { + return strings.Compare(s[a].EventID, s[b].EventID) < 0 +} + +func (s StateAtEventAndReferences) Len() int { + return len(s) +} + +func (s StateAtEventAndReferences) Swap(a, b int) { + s[a], s[b] = s[b], s[a] +} + // An Event is a gomatrixserverlib.Event with the numeric event ID attached. // It is when performing bulk event lookup in the database. type Event struct { diff --git a/roomserver/types/types_test.go b/roomserver/types/types_test.go index b1e84b821..a26b80f74 100644 --- a/roomserver/types/types_test.go +++ b/roomserver/types/types_test.go @@ -1,6 +1,7 @@ package types import ( + "sort" "testing" ) @@ -24,3 +25,66 @@ func TestDeduplicateStateEntries(t *testing.T) { } } } + +func TestStateKeyTupleSorter(t *testing.T) { + input := StateKeyTupleSorter{ + {EventTypeNID: 1, EventStateKeyNID: 2}, + {EventTypeNID: 1, EventStateKeyNID: 4}, + {EventTypeNID: 2, EventStateKeyNID: 2}, + {EventTypeNID: 1, EventStateKeyNID: 1}, + } + want := []StateKeyTuple{ + {EventTypeNID: 1, EventStateKeyNID: 1}, + {EventTypeNID: 1, EventStateKeyNID: 2}, + {EventTypeNID: 1, EventStateKeyNID: 4}, + {EventTypeNID: 2, EventStateKeyNID: 2}, + } + doNotWant := []StateKeyTuple{ + {EventTypeNID: 0, EventStateKeyNID: 0}, + {EventTypeNID: 1, EventStateKeyNID: 3}, + {EventTypeNID: 2, EventStateKeyNID: 1}, + {EventTypeNID: 3, EventStateKeyNID: 1}, + } + wantTypeNIDs := []int64{1, 2} + wantStateKeyNIDs := []int64{1, 2, 4} + + // Sort the input and check it's in the right order. + sort.Sort(input) + gotTypeNIDs, gotStateKeyNIDs := input.TypesAndStateKeysAsArrays() + + for i := range want { + if input[i] != want[i] { + t.Errorf("Wanted %#v at index %d got %#v", want[i], i, input[i]) + } + + if !input.contains(want[i]) { + t.Errorf("Wanted %#v.contains(%#v) to be true but got false", input, want[i]) + } + } + + for i := range doNotWant { + if input.contains(doNotWant[i]) { + t.Errorf("Wanted %#v.contains(%#v) to be false but got true", input, doNotWant[i]) + } + } + + if len(wantTypeNIDs) != len(gotTypeNIDs) { + t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs) + } + + for i := range wantTypeNIDs { + if wantTypeNIDs[i] != gotTypeNIDs[i] { + t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs) + } + } + + if len(wantStateKeyNIDs) != len(gotStateKeyNIDs) { + t.Fatalf("Wanted state key NIDs %#v got %#v", wantStateKeyNIDs, gotStateKeyNIDs) + } + + for i := range wantStateKeyNIDs { + if wantStateKeyNIDs[i] != gotStateKeyNIDs[i] { + t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs) + } + } +} diff --git a/setup/base/base.go b/setup/base/base.go index cd96ea1be..740fa7c9b 100644 --- a/setup/base/base.go +++ b/setup/base/base.go @@ -86,6 +86,7 @@ type BaseDendrite struct { DNSCache *gomatrixserverlib.DNSCache Database *sql.DB DatabaseWriter sqlutil.Writer + EnableMetrics bool } const NoListener = "" @@ -96,7 +97,7 @@ const HTTPClientTimeout = time.Second * 30 type BaseDendriteOptions int const ( - NoCacheMetrics BaseDendriteOptions = iota + DisableMetrics BaseDendriteOptions = iota UseHTTPAPIs PolylithMode ) @@ -107,12 +108,12 @@ const ( func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...BaseDendriteOptions) *BaseDendrite { platformSanityChecks() useHTTPAPIs := false - cacheMetrics := true + enableMetrics := true isMonolith := true for _, opt := range options { switch opt { - case NoCacheMetrics: - cacheMetrics = false + case DisableMetrics: + enableMetrics = false case UseHTTPAPIs: useHTTPAPIs = true case PolylithMode: @@ -160,7 +161,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base } } - cache, err := caching.NewInMemoryLRUCache(cacheMetrics) + cache, err := caching.NewInMemoryLRUCache(enableMetrics) if err != nil { logrus.WithError(err).Warnf("Failed to create cache") } @@ -246,6 +247,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base apiHttpClient: &apiClient, Database: db, // set if monolith with global connection pool only DatabaseWriter: writer, // set if monolith with global connection pool only + EnableMetrics: enableMetrics, } } diff --git a/setup/config/config_appservice.go b/setup/config/config_appservice.go index d93b6ebe0..ff3287714 100644 --- a/setup/config/config_appservice.go +++ b/setup/config/config_appservice.go @@ -50,11 +50,14 @@ func (c *AppServiceAPI) Defaults(generate bool) { } func (c *AppServiceAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { - checkURL(configErrs, "app_service_api.internal_api.listen", string(c.InternalAPI.Listen)) - checkURL(configErrs, "app_service_api.internal_api.bind", string(c.InternalAPI.Connect)) if c.Matrix.DatabaseOptions.ConnectionString == "" { checkNotEmpty(configErrs, "app_service_api.database.connection_string", string(c.Database.ConnectionString)) } + if isMonolith { // polylith required configs below + return + } + checkURL(configErrs, "app_service_api.internal_api.listen", string(c.InternalAPI.Listen)) + checkURL(configErrs, "app_service_api.internal_api.connect", string(c.InternalAPI.Connect)) } // ApplicationServiceNamespace is the namespace that a specific application diff --git a/setup/config/config_clientapi.go b/setup/config/config_clientapi.go index 6104ed8b9..bb786a145 100644 --- a/setup/config/config_clientapi.go +++ b/setup/config/config_clientapi.go @@ -67,19 +67,13 @@ func (c *ClientAPI) Defaults(generate bool) { } func (c *ClientAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { - checkURL(configErrs, "client_api.internal_api.listen", string(c.InternalAPI.Listen)) - checkURL(configErrs, "client_api.internal_api.connect", string(c.InternalAPI.Connect)) - if !isMonolith { - checkURL(configErrs, "client_api.external_api.listen", string(c.ExternalAPI.Listen)) - } - if c.RecaptchaEnabled { - checkNotEmpty(configErrs, "client_api.recaptcha_public_key", string(c.RecaptchaPublicKey)) - checkNotEmpty(configErrs, "client_api.recaptcha_private_key", string(c.RecaptchaPrivateKey)) - checkNotEmpty(configErrs, "client_api.recaptcha_siteverify_api", string(c.RecaptchaSiteVerifyAPI)) - } c.TURN.Verify(configErrs) c.RateLimiting.Verify(configErrs) - + if c.RecaptchaEnabled { + checkNotEmpty(configErrs, "client_api.recaptcha_public_key", c.RecaptchaPublicKey) + checkNotEmpty(configErrs, "client_api.recaptcha_private_key", c.RecaptchaPrivateKey) + checkNotEmpty(configErrs, "client_api.recaptcha_siteverify_api", c.RecaptchaSiteVerifyAPI) + } // Ensure there is any spam counter measure when enabling registration if !c.RegistrationDisabled && !c.OpenRegistrationWithoutVerificationEnabled { if !c.RecaptchaEnabled { @@ -93,6 +87,12 @@ func (c *ClientAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { ) } } + if isMonolith { // polylith required configs below + return + } + checkURL(configErrs, "client_api.internal_api.listen", string(c.InternalAPI.Listen)) + checkURL(configErrs, "client_api.internal_api.connect", string(c.InternalAPI.Connect)) + checkURL(configErrs, "client_api.external_api.listen", string(c.ExternalAPI.Listen)) } type TURN struct { diff --git a/setup/config/config_federationapi.go b/setup/config/config_federationapi.go index f62a23e1f..a7a515fda 100644 --- a/setup/config/config_federationapi.go +++ b/setup/config/config_federationapi.go @@ -34,24 +34,24 @@ func (c *FederationAPI) Defaults(generate bool) { c.InternalAPI.Listen = "http://localhost:7772" c.InternalAPI.Connect = "http://localhost:7772" c.ExternalAPI.Listen = "http://[::]:8072" + c.FederationMaxRetries = 16 + c.DisableTLSValidation = false c.Database.Defaults(10) if generate { c.Database.ConnectionString = "file:federationapi.db" } - - c.FederationMaxRetries = 16 - c.DisableTLSValidation = false } func (c *FederationAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { - checkURL(configErrs, "federation_api.internal_api.listen", string(c.InternalAPI.Listen)) - checkURL(configErrs, "federation_api.internal_api.connect", string(c.InternalAPI.Connect)) - if !isMonolith { - checkURL(configErrs, "federation_api.external_api.listen", string(c.ExternalAPI.Listen)) - } if c.Matrix.DatabaseOptions.ConnectionString == "" { checkNotEmpty(configErrs, "federation_api.database.connection_string", string(c.Database.ConnectionString)) } + if isMonolith { // polylith required configs below + return + } + checkURL(configErrs, "federation_api.external_api.listen", string(c.ExternalAPI.Listen)) + checkURL(configErrs, "federation_api.internal_api.listen", string(c.InternalAPI.Listen)) + checkURL(configErrs, "federation_api.internal_api.connect", string(c.InternalAPI.Connect)) } // The config for setting a proxy to use for server->server requests diff --git a/setup/config/config_jetstream.go b/setup/config/config_jetstream.go index b6a93d398..e4cfd4d3b 100644 --- a/setup/config/config_jetstream.go +++ b/setup/config/config_jetstream.go @@ -36,9 +36,10 @@ func (c *JetStream) Defaults(generate bool) { } func (c *JetStream) Verify(configErrs *ConfigErrors, isMonolith bool) { + if isMonolith { // polylith required configs below + return + } // If we are running in a polylith deployment then we need at least // one NATS JetStream server to talk to. - if !isMonolith { - checkNotZero(configErrs, "global.jetstream.addresses", int64(len(c.Addresses))) - } + checkNotZero(configErrs, "global.jetstream.addresses", int64(len(c.Addresses))) } diff --git a/setup/config/config_keyserver.go b/setup/config/config_keyserver.go index 9e2d54cdc..5f2f22c8a 100644 --- a/setup/config/config_keyserver.go +++ b/setup/config/config_keyserver.go @@ -18,9 +18,12 @@ func (c *KeyServer) Defaults(generate bool) { } func (c *KeyServer) Verify(configErrs *ConfigErrors, isMonolith bool) { - checkURL(configErrs, "key_server.internal_api.listen", string(c.InternalAPI.Listen)) - checkURL(configErrs, "key_server.internal_api.bind", string(c.InternalAPI.Connect)) if c.Matrix.DatabaseOptions.ConnectionString == "" { checkNotEmpty(configErrs, "key_server.database.connection_string", string(c.Database.ConnectionString)) } + if isMonolith { // polylith required configs below + return + } + checkURL(configErrs, "key_server.internal_api.listen", string(c.InternalAPI.Listen)) + checkURL(configErrs, "key_server.internal_api.connect", string(c.InternalAPI.Connect)) } diff --git a/setup/config/config_mediaapi.go b/setup/config/config_mediaapi.go index 273de322a..9717aa59e 100644 --- a/setup/config/config_mediaapi.go +++ b/setup/config/config_mediaapi.go @@ -42,26 +42,19 @@ func (c *MediaAPI) Defaults(generate bool) { c.InternalAPI.Listen = "http://localhost:7774" c.InternalAPI.Connect = "http://localhost:7774" c.ExternalAPI.Listen = "http://[::]:8074" + c.MaxFileSizeBytes = DefaultMaxFileSizeBytes + c.MaxThumbnailGenerators = 10 c.Database.Defaults(5) if generate { c.Database.ConnectionString = "file:mediaapi.db" c.BasePath = "./media_store" } - - c.MaxFileSizeBytes = DefaultMaxFileSizeBytes - c.MaxThumbnailGenerators = 10 } func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { - checkURL(configErrs, "media_api.internal_api.listen", string(c.InternalAPI.Listen)) - checkURL(configErrs, "media_api.internal_api.connect", string(c.InternalAPI.Connect)) - if !isMonolith { - checkURL(configErrs, "media_api.external_api.listen", string(c.ExternalAPI.Listen)) - } if c.Matrix.DatabaseOptions.ConnectionString == "" { checkNotEmpty(configErrs, "media_api.database.connection_string", string(c.Database.ConnectionString)) } - checkNotEmpty(configErrs, "media_api.base_path", string(c.BasePath)) checkPositive(configErrs, "media_api.max_file_size_bytes", int64(c.MaxFileSizeBytes)) checkPositive(configErrs, "media_api.max_thumbnail_generators", int64(c.MaxThumbnailGenerators)) @@ -70,4 +63,10 @@ func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { checkPositive(configErrs, fmt.Sprintf("media_api.thumbnail_sizes[%d].width", i), int64(size.Width)) checkPositive(configErrs, fmt.Sprintf("media_api.thumbnail_sizes[%d].height", i), int64(size.Height)) } + if isMonolith { // polylith required configs below + return + } + checkURL(configErrs, "media_api.internal_api.listen", string(c.InternalAPI.Listen)) + checkURL(configErrs, "media_api.internal_api.connect", string(c.InternalAPI.Connect)) + checkURL(configErrs, "media_api.external_api.listen", string(c.ExternalAPI.Listen)) } diff --git a/setup/config/config_roomserver.go b/setup/config/config_roomserver.go index 8a3227349..bd6aa1167 100644 --- a/setup/config/config_roomserver.go +++ b/setup/config/config_roomserver.go @@ -18,9 +18,12 @@ func (c *RoomServer) Defaults(generate bool) { } func (c *RoomServer) Verify(configErrs *ConfigErrors, isMonolith bool) { - checkURL(configErrs, "room_server.internal_api.listen", string(c.InternalAPI.Listen)) - checkURL(configErrs, "room_server.internal_ap.bind", string(c.InternalAPI.Connect)) if c.Matrix.DatabaseOptions.ConnectionString == "" { checkNotEmpty(configErrs, "room_server.database.connection_string", string(c.Database.ConnectionString)) } + if isMonolith { // polylith required configs below + return + } + checkURL(configErrs, "room_server.internal_api.listen", string(c.InternalAPI.Listen)) + checkURL(configErrs, "room_server.internal_ap.connect", string(c.InternalAPI.Connect)) } diff --git a/setup/config/config_syncapi.go b/setup/config/config_syncapi.go index 48fd9f506..7d5e3808a 100644 --- a/setup/config/config_syncapi.go +++ b/setup/config/config_syncapi.go @@ -22,12 +22,13 @@ func (c *SyncAPI) Defaults(generate bool) { } func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { - checkURL(configErrs, "sync_api.internal_api.listen", string(c.InternalAPI.Listen)) - checkURL(configErrs, "sync_api.internal_api.bind", string(c.InternalAPI.Connect)) - if !isMonolith { - checkURL(configErrs, "sync_api.external_api.listen", string(c.ExternalAPI.Listen)) - } if c.Matrix.DatabaseOptions.ConnectionString == "" { checkNotEmpty(configErrs, "sync_api.database", string(c.Database.ConnectionString)) } + if isMonolith { // polylith required configs below + return + } + checkURL(configErrs, "sync_api.internal_api.listen", string(c.InternalAPI.Listen)) + checkURL(configErrs, "sync_api.internal_api.connect", string(c.InternalAPI.Connect)) + checkURL(configErrs, "sync_api.external_api.listen", string(c.ExternalAPI.Listen)) } diff --git a/setup/config/config_userapi.go b/setup/config/config_userapi.go index 4aa3b57bb..d1e2b7fe1 100644 --- a/setup/config/config_userapi.go +++ b/setup/config/config_userapi.go @@ -26,19 +26,22 @@ const DefaultOpenIDTokenLifetimeMS = 3600000 // 60 minutes func (c *UserAPI) Defaults(generate bool) { c.InternalAPI.Listen = "http://localhost:7781" c.InternalAPI.Connect = "http://localhost:7781" + c.BCryptCost = bcrypt.DefaultCost + c.OpenIDTokenLifetimeMS = DefaultOpenIDTokenLifetimeMS c.AccountDatabase.Defaults(10) if generate { c.AccountDatabase.ConnectionString = "file:userapi_accounts.db" } - c.BCryptCost = bcrypt.DefaultCost - c.OpenIDTokenLifetimeMS = DefaultOpenIDTokenLifetimeMS } func (c *UserAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { - checkURL(configErrs, "user_api.internal_api.listen", string(c.InternalAPI.Listen)) - checkURL(configErrs, "user_api.internal_api.connect", string(c.InternalAPI.Connect)) + checkPositive(configErrs, "user_api.openid_token_lifetime_ms", c.OpenIDTokenLifetimeMS) if c.Matrix.DatabaseOptions.ConnectionString == "" { checkNotEmpty(configErrs, "user_api.account_database.connection_string", string(c.AccountDatabase.ConnectionString)) } - checkPositive(configErrs, "user_api.openid_token_lifetime_ms", c.OpenIDTokenLifetimeMS) + if isMonolith { // polylith required configs below + return + } + checkURL(configErrs, "user_api.internal_api.listen", string(c.InternalAPI.Listen)) + checkURL(configErrs, "user_api.internal_api.connect", string(c.InternalAPI.Connect)) } diff --git a/setup/jetstream/nats.go b/setup/jetstream/nats.go index 426f02bb6..248b0e656 100644 --- a/setup/jetstream/nats.go +++ b/setup/jetstream/nats.go @@ -13,6 +13,7 @@ import ( "github.com/sirupsen/logrus" natsserver "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" natsclient "github.com/nats-io/nats.go" ) @@ -21,6 +22,13 @@ type NATSInstance struct { sync.Mutex } +func DeleteAllStreams(js nats.JetStreamContext, cfg *config.JetStream) { + for _, stream := range streams { // streams are defined in streams.go + name := cfg.Prefixed(stream.Name) + _ = js.DeleteStream(name) + } +} + func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetStream) (natsclient.JetStreamContext, *natsclient.Conn) { // check if we need an in-process NATS Server if len(cfg.Addresses) != 0 { diff --git a/syncapi/consumers/presence.go b/syncapi/consumers/presence.go index 388c08ff4..bfd72d604 100644 --- a/syncapi/consumers/presence.go +++ b/syncapi/consumers/presence.go @@ -138,9 +138,12 @@ func (s *PresenceConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool { presence := msg.Header.Get("presence") timestamp := msg.Header.Get("last_active_ts") fromSync, _ := strconv.ParseBool(msg.Header.Get("from_sync")) - logrus.Debugf("syncAPI received presence event: %+v", msg.Header) + if fromSync { // do not process local presence changes; we already did this synchronously. + return true + } + ts, err := strconv.Atoi(timestamp) if err != nil { return true @@ -151,15 +154,19 @@ func (s *PresenceConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool { newMsg := msg.Header.Get("status_msg") statusMsg = &newMsg } - // OK is already checked, so no need to do it again + // already checked, so no need to check error p, _ := types.PresenceFromString(presence) - pos, err := s.db.UpdatePresence(ctx, userID, p, statusMsg, gomatrixserverlib.Timestamp(ts), fromSync) - if err != nil { - return true - } - - s.stream.Advance(pos) - s.notifier.OnNewPresence(types.StreamingToken{PresencePosition: pos}, userID) + s.EmitPresence(ctx, userID, p, statusMsg, ts, fromSync) return true } + +func (s *PresenceConsumer) EmitPresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, ts int, fromSync bool) { + pos, err := s.db.UpdatePresence(ctx, userID, presence, statusMsg, gomatrixserverlib.Timestamp(ts), fromSync) + if err != nil { + logrus.WithError(err).WithField("user", userID).WithField("presence", presence).Warn("failed to updated presence for user") + return + } + s.stream.Advance(pos) + s.notifier.OnNewPresence(types.StreamingToken{PresencePosition: pos}, userID) +} diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 63bde8166..f0ca2106f 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -347,9 +347,11 @@ func (s *OutputRoomEventConsumer) onNewInviteEvent( ctx context.Context, msg api.OutputNewInviteEvent, ) { if msg.Event.StateKey() == nil { - log.WithFields(log.Fields{ - "event": string(msg.Event.JSON()), - }).Panicf("roomserver output log: invite has no state key") + return + } + if _, serverName, err := gomatrixserverlib.SplitID('@', *msg.Event.StateKey()); err != nil { + return + } else if serverName != s.cfg.Matrix.ServerName { return } pduPos, err := s.db.AddInviteEvent(ctx, msg.Event) diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go index 87cc2aae0..96438e184 100644 --- a/syncapi/routing/context.go +++ b/syncapi/routing/context.go @@ -73,6 +73,12 @@ func Context( logrus.WithError(err).Error("unable to query membership") return jsonerror.InternalServerError() } + if !membershipRes.RoomExists { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("room does not exist"), + } + } stateFilter := gomatrixserverlib.StateFilter{ Limit: 100, diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index b0c990ec0..e55c661d6 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -68,10 +68,16 @@ func OnIncomingMessagesRequest( var err error // check if the user has already forgotten about this room - isForgotten, err := checkIsRoomForgotten(req.Context(), roomID, device.UserID, rsAPI) + isForgotten, roomExists, err := checkIsRoomForgotten(req.Context(), roomID, device.UserID, rsAPI) if err != nil { return jsonerror.InternalServerError() } + if !roomExists { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("room does not exist"), + } + } if isForgotten { return util.JSONResponse{ @@ -244,17 +250,17 @@ func OnIncomingMessagesRequest( } } -func checkIsRoomForgotten(ctx context.Context, roomID, userID string, rsAPI api.SyncRoomserverAPI) (bool, error) { +func checkIsRoomForgotten(ctx context.Context, roomID, userID string, rsAPI api.SyncRoomserverAPI) (forgotten bool, exists bool, err error) { req := api.QueryMembershipForUserRequest{ RoomID: roomID, UserID: userID, } resp := api.QueryMembershipForUserResponse{} if err := rsAPI.QueryMembershipForUser(ctx, &req, &resp); err != nil { - return false, err + return false, false, err } - return resp.IsRoomForgotten, nil + return resp.IsRoomForgotten, resp.RoomExists, nil } // retrieveEvents retrieves events from the local database for a request on diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index 17e2feab6..d84d0cfa2 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -69,6 +69,11 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( -- were emitted. exclude_from_sync BOOL DEFAULT FALSE ); + +CREATE INDEX IF NOT EXISTS syncapi_output_room_events_type_idx ON syncapi_output_room_events (type); +CREATE INDEX IF NOT EXISTS syncapi_output_room_events_sender_idx ON syncapi_output_room_events (sender); +CREATE INDEX IF NOT EXISTS syncapi_output_room_events_room_id_idx ON syncapi_output_room_events (room_id); +CREATE INDEX IF NOT EXISTS syncapi_output_room_events_exclude_from_sync_idx ON syncapi_output_room_events (exclude_from_sync); ` const insertEventSQL = "" + diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 188f7582b..f9961a9d1 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -49,6 +49,11 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( transaction_id TEXT, exclude_from_sync BOOL NOT NULL DEFAULT FALSE ); + +CREATE INDEX IF NOT EXISTS syncapi_output_room_events_type_idx ON syncapi_output_room_events (type); +CREATE INDEX IF NOT EXISTS syncapi_output_room_events_sender_idx ON syncapi_output_room_events (sender); +CREATE INDEX IF NOT EXISTS syncapi_output_room_events_room_id_idx ON syncapi_output_room_events (room_id); +CREATE INDEX IF NOT EXISTS syncapi_output_room_events_exclude_from_sync_idx ON syncapi_output_room_events (exclude_from_sync); ` const insertEventSQL = "" + diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index 1150c2f3d..563c92e34 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -47,7 +47,7 @@ func MustWriteEvents(t *testing.T, db storage.Database, events []*gomatrixserver func TestWriteEvents(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - alice := test.NewUser() + alice := test.NewUser(t) r := test.NewRoom(t, alice) db, close := MustCreateDatabase(t, dbType) defer close() @@ -60,7 +60,7 @@ func TestRecentEventsPDU(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { db, close := MustCreateDatabase(t, dbType) defer close() - alice := test.NewUser() + alice := test.NewUser(t) // dummy room to make sure SQL queries are filtering on room ID MustWriteEvents(t, db, test.NewRoom(t, alice).Events()) @@ -163,7 +163,7 @@ func TestGetEventsInRangeWithTopologyToken(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { db, close := MustCreateDatabase(t, dbType) defer close() - alice := test.NewUser() + alice := test.NewUser(t) r := test.NewRoom(t, alice) for i := 0; i < 10; i++ { r.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": fmt.Sprintf("hi %d", i)}) diff --git a/syncapi/storage/tables/output_room_events_test.go b/syncapi/storage/tables/output_room_events_test.go index 8bbf879d4..69bbd04c9 100644 --- a/syncapi/storage/tables/output_room_events_test.go +++ b/syncapi/storage/tables/output_room_events_test.go @@ -45,7 +45,7 @@ func newOutputRoomEventsTable(t *testing.T, dbType test.DBType) (tables.Events, func TestOutputRoomEventsTable(t *testing.T) { ctx := context.Background() - alice := test.NewUser() + alice := test.NewUser(t) room := test.NewRoom(t, alice) test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { tab, db, close := newOutputRoomEventsTable(t, dbType) diff --git a/syncapi/storage/tables/topology_test.go b/syncapi/storage/tables/topology_test.go index 2334aae2e..f4f75bdf3 100644 --- a/syncapi/storage/tables/topology_test.go +++ b/syncapi/storage/tables/topology_test.go @@ -40,7 +40,7 @@ func newTopologyTable(t *testing.T, dbType test.DBType) (tables.Topology, *sql.D func TestTopologyTable(t *testing.T) { ctx := context.Background() - alice := test.NewUser() + alice := test.NewUser(t) room := test.NewRoom(t, alice) test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { tab, db, close := newTopologyTable(t, dbType) diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 99d1e40c3..7b9526b53 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -53,23 +53,30 @@ type RequestPool struct { streams *streams.Streams Notifier *notifier.Notifier producer PresencePublisher + consumer PresenceConsumer } type PresencePublisher interface { SendPresence(userID string, presence types.Presence, statusMsg *string) error } +type PresenceConsumer interface { + EmitPresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, ts int, fromSync bool) +} + // NewRequestPool makes a new RequestPool func NewRequestPool( db storage.Database, cfg *config.SyncAPI, userAPI userapi.SyncUserAPI, keyAPI keyapi.SyncKeyAPI, rsAPI roomserverAPI.SyncRoomserverAPI, streams *streams.Streams, notifier *notifier.Notifier, - producer PresencePublisher, + producer PresencePublisher, consumer PresenceConsumer, enableMetrics bool, ) *RequestPool { - prometheus.MustRegister( - activeSyncRequests, waitingSyncRequests, - ) + if enableMetrics { + prometheus.MustRegister( + activeSyncRequests, waitingSyncRequests, + ) + } rp := &RequestPool{ db: db, cfg: cfg, @@ -81,6 +88,7 @@ func NewRequestPool( streams: streams, Notifier: notifier, producer: producer, + consumer: consumer, } go rp.cleanLastSeen() go rp.cleanPresence(db, time.Minute*5) @@ -158,6 +166,13 @@ func (rp *RequestPool) updatePresence(db storage.Presence, presence string, user logrus.WithError(err).Error("Unable to publish presence message from sync") return } + + // now synchronously update our view of the world. It's critical we do this before calculating + // the /sync response else we may not return presence: online immediately. + rp.consumer.EmitPresence( + context.Background(), userID, presenceID, newPresence.ClientFields.StatusMsg, + int(gomatrixserverlib.AsTimestamp(time.Now())), true, + ) } func (rp *RequestPool) updateLastSeen(req *http.Request, device *userapi.Device) { @@ -236,114 +251,151 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. waitingSyncRequests.Inc() defer waitingSyncRequests.Dec() - currentPos := rp.Notifier.CurrentPosition() + // loop until we get some data + for { + startTime := time.Now() + currentPos := rp.Notifier.CurrentPosition() - if !rp.shouldReturnImmediately(syncReq, currentPos) { - timer := time.NewTimer(syncReq.Timeout) // case of timeout=0 is handled above - defer timer.Stop() + // if the since token matches the current positions, wait via the notifier + if !rp.shouldReturnImmediately(syncReq, currentPos) { + timer := time.NewTimer(syncReq.Timeout) // case of timeout=0 is handled above + defer timer.Stop() - userStreamListener := rp.Notifier.GetListener(*syncReq) - defer userStreamListener.Close() + userStreamListener := rp.Notifier.GetListener(*syncReq) + defer userStreamListener.Close() - giveup := func() util.JSONResponse { - syncReq.Response.NextBatch = syncReq.Since - return util.JSONResponse{ - Code: http.StatusOK, - JSON: syncReq.Response, + giveup := func() util.JSONResponse { + syncReq.Log.Debugln("Responding to sync since client gave up or timeout was reached") + syncReq.Response.NextBatch = syncReq.Since + // We should always try to include OTKs in sync responses, otherwise clients might upload keys + // even if that's not required. See also: + // https://github.com/matrix-org/synapse/blob/29f06704b8871a44926f7c99e73cf4a978fb8e81/synapse/rest/client/sync.py#L276-L281 + // Only try to get OTKs if the context isn't already done. + if syncReq.Context.Err() == nil { + err = internal.DeviceOTKCounts(syncReq.Context, rp.keyAPI, syncReq.Device.UserID, syncReq.Device.ID, syncReq.Response) + if err != nil && err != context.Canceled { + syncReq.Log.WithError(err).Warn("failed to get OTK counts") + } + } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: syncReq.Response, + } + } + + select { + case <-syncReq.Context.Done(): // Caller gave up + return giveup() + + case <-timer.C: // Timeout reached + return giveup() + + case <-userStreamListener.GetNotifyChannel(syncReq.Since): + syncReq.Log.Debugln("Responding to sync after wake-up") + currentPos.ApplyUpdates(userStreamListener.GetSyncPosition()) + } + } else { + syncReq.Log.WithField("currentPos", currentPos).Debugln("Responding to sync immediately") + } + + if syncReq.Since.IsEmpty() { + // Complete sync + syncReq.Response.NextBatch = types.StreamingToken{ + PDUPosition: rp.streams.PDUStreamProvider.CompleteSync( + syncReq.Context, syncReq, + ), + TypingPosition: rp.streams.TypingStreamProvider.CompleteSync( + syncReq.Context, syncReq, + ), + ReceiptPosition: rp.streams.ReceiptStreamProvider.CompleteSync( + syncReq.Context, syncReq, + ), + InvitePosition: rp.streams.InviteStreamProvider.CompleteSync( + syncReq.Context, syncReq, + ), + SendToDevicePosition: rp.streams.SendToDeviceStreamProvider.CompleteSync( + syncReq.Context, syncReq, + ), + AccountDataPosition: rp.streams.AccountDataStreamProvider.CompleteSync( + syncReq.Context, syncReq, + ), + NotificationDataPosition: rp.streams.NotificationDataStreamProvider.CompleteSync( + syncReq.Context, syncReq, + ), + DeviceListPosition: rp.streams.DeviceListStreamProvider.CompleteSync( + syncReq.Context, syncReq, + ), + PresencePosition: rp.streams.PresenceStreamProvider.CompleteSync( + syncReq.Context, syncReq, + ), + } + } else { + // Incremental sync + syncReq.Response.NextBatch = types.StreamingToken{ + PDUPosition: rp.streams.PDUStreamProvider.IncrementalSync( + syncReq.Context, syncReq, + syncReq.Since.PDUPosition, currentPos.PDUPosition, + ), + TypingPosition: rp.streams.TypingStreamProvider.IncrementalSync( + syncReq.Context, syncReq, + syncReq.Since.TypingPosition, currentPos.TypingPosition, + ), + ReceiptPosition: rp.streams.ReceiptStreamProvider.IncrementalSync( + syncReq.Context, syncReq, + syncReq.Since.ReceiptPosition, currentPos.ReceiptPosition, + ), + InvitePosition: rp.streams.InviteStreamProvider.IncrementalSync( + syncReq.Context, syncReq, + syncReq.Since.InvitePosition, currentPos.InvitePosition, + ), + SendToDevicePosition: rp.streams.SendToDeviceStreamProvider.IncrementalSync( + syncReq.Context, syncReq, + syncReq.Since.SendToDevicePosition, currentPos.SendToDevicePosition, + ), + AccountDataPosition: rp.streams.AccountDataStreamProvider.IncrementalSync( + syncReq.Context, syncReq, + syncReq.Since.AccountDataPosition, currentPos.AccountDataPosition, + ), + NotificationDataPosition: rp.streams.NotificationDataStreamProvider.IncrementalSync( + syncReq.Context, syncReq, + syncReq.Since.NotificationDataPosition, currentPos.NotificationDataPosition, + ), + DeviceListPosition: rp.streams.DeviceListStreamProvider.IncrementalSync( + syncReq.Context, syncReq, + syncReq.Since.DeviceListPosition, currentPos.DeviceListPosition, + ), + PresencePosition: rp.streams.PresenceStreamProvider.IncrementalSync( + syncReq.Context, syncReq, + syncReq.Since.PresencePosition, currentPos.PresencePosition, + ), + } + // it's possible for there to be no updates for this user even though since < current pos, + // e.g busy servers with a quiet user. In this scenario, we don't want to return a no-op + // response immediately, so let's try this again but pretend they bumped their since token. + // If the incremental sync was processed very quickly then we expect the next loop to block + // with a notifier, but if things are slow it's entirely possible that currentPos is no + // longer the current position so we will hit this code path again. We need to do this and + // not return a no-op response because: + // - It's an inefficient use of bandwidth. + // - Some sytests which test 'waking up' sync rely on some sync requests to block, which + // they weren't always doing, resulting in flakey tests. + if !syncReq.Response.HasUpdates() { + syncReq.Since = currentPos + // do not loop again if the ?timeout= is 0 as that means "return immediately" + if syncReq.Timeout > 0 { + syncReq.Timeout = syncReq.Timeout - time.Since(startTime) + if syncReq.Timeout < 0 { + syncReq.Timeout = 0 + } + continue + } } } - select { - case <-syncReq.Context.Done(): // Caller gave up - return giveup() - - case <-timer.C: // Timeout reached - return giveup() - - case <-userStreamListener.GetNotifyChannel(syncReq.Since): - syncReq.Log.Debugln("Responding to sync after wake-up") - currentPos.ApplyUpdates(userStreamListener.GetSyncPosition()) + return util.JSONResponse{ + Code: http.StatusOK, + JSON: syncReq.Response, } - } else { - syncReq.Log.WithField("currentPos", currentPos).Debugln("Responding to sync immediately") - } - - if syncReq.Since.IsEmpty() { - // Complete sync - syncReq.Response.NextBatch = types.StreamingToken{ - PDUPosition: rp.streams.PDUStreamProvider.CompleteSync( - syncReq.Context, syncReq, - ), - TypingPosition: rp.streams.TypingStreamProvider.CompleteSync( - syncReq.Context, syncReq, - ), - ReceiptPosition: rp.streams.ReceiptStreamProvider.CompleteSync( - syncReq.Context, syncReq, - ), - InvitePosition: rp.streams.InviteStreamProvider.CompleteSync( - syncReq.Context, syncReq, - ), - SendToDevicePosition: rp.streams.SendToDeviceStreamProvider.CompleteSync( - syncReq.Context, syncReq, - ), - AccountDataPosition: rp.streams.AccountDataStreamProvider.CompleteSync( - syncReq.Context, syncReq, - ), - NotificationDataPosition: rp.streams.NotificationDataStreamProvider.CompleteSync( - syncReq.Context, syncReq, - ), - DeviceListPosition: rp.streams.DeviceListStreamProvider.CompleteSync( - syncReq.Context, syncReq, - ), - PresencePosition: rp.streams.PresenceStreamProvider.CompleteSync( - syncReq.Context, syncReq, - ), - } - } else { - // Incremental sync - syncReq.Response.NextBatch = types.StreamingToken{ - PDUPosition: rp.streams.PDUStreamProvider.IncrementalSync( - syncReq.Context, syncReq, - syncReq.Since.PDUPosition, currentPos.PDUPosition, - ), - TypingPosition: rp.streams.TypingStreamProvider.IncrementalSync( - syncReq.Context, syncReq, - syncReq.Since.TypingPosition, currentPos.TypingPosition, - ), - ReceiptPosition: rp.streams.ReceiptStreamProvider.IncrementalSync( - syncReq.Context, syncReq, - syncReq.Since.ReceiptPosition, currentPos.ReceiptPosition, - ), - InvitePosition: rp.streams.InviteStreamProvider.IncrementalSync( - syncReq.Context, syncReq, - syncReq.Since.InvitePosition, currentPos.InvitePosition, - ), - SendToDevicePosition: rp.streams.SendToDeviceStreamProvider.IncrementalSync( - syncReq.Context, syncReq, - syncReq.Since.SendToDevicePosition, currentPos.SendToDevicePosition, - ), - AccountDataPosition: rp.streams.AccountDataStreamProvider.IncrementalSync( - syncReq.Context, syncReq, - syncReq.Since.AccountDataPosition, currentPos.AccountDataPosition, - ), - NotificationDataPosition: rp.streams.NotificationDataStreamProvider.IncrementalSync( - syncReq.Context, syncReq, - syncReq.Since.NotificationDataPosition, currentPos.NotificationDataPosition, - ), - DeviceListPosition: rp.streams.DeviceListStreamProvider.IncrementalSync( - syncReq.Context, syncReq, - syncReq.Since.DeviceListPosition, currentPos.DeviceListPosition, - ), - PresencePosition: rp.streams.PresenceStreamProvider.IncrementalSync( - syncReq.Context, syncReq, - syncReq.Since.PresencePosition, currentPos.PresencePosition, - ), - } - } - - return util.JSONResponse{ - Code: http.StatusOK, - JSON: syncReq.Response, } } diff --git a/syncapi/sync/requestpool_test.go b/syncapi/sync/requestpool_test.go index 5e52bc7c9..0c7209521 100644 --- a/syncapi/sync/requestpool_test.go +++ b/syncapi/sync/requestpool_test.go @@ -38,6 +38,12 @@ func (d dummyDB) MaxStreamPositionForPresence(ctx context.Context) (types.Stream return 0, nil } +type dummyConsumer struct{} + +func (d dummyConsumer) EmitPresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, ts int, fromSync bool) { + +} + func TestRequestPool_updatePresence(t *testing.T) { type args struct { presence string @@ -45,6 +51,7 @@ func TestRequestPool_updatePresence(t *testing.T) { sleep time.Duration } publisher := &dummyPublisher{} + consumer := &dummyConsumer{} syncMap := sync.Map{} tests := []struct { @@ -101,6 +108,7 @@ func TestRequestPool_updatePresence(t *testing.T) { rp := &RequestPool{ presence: &syncMap, producer: publisher, + consumer: consumer, cfg: &config.SyncAPI{ Matrix: &config.Global{ JetStream: config.JetStream{ diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index dbc6e240c..92db18d56 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -64,8 +64,17 @@ func AddPublicRoutes( Topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent), JetStream: js, } + presenceConsumer := consumers.NewPresenceConsumer( + base.ProcessContext, cfg, js, natsClient, syncDB, + notifier, streams.PresenceStreamProvider, + userAPI, + ) - requestPool := sync.NewRequestPool(syncDB, cfg, userAPI, keyAPI, rsAPI, streams, notifier, federationPresenceProducer) + requestPool := sync.NewRequestPool(syncDB, cfg, userAPI, keyAPI, rsAPI, streams, notifier, federationPresenceProducer, presenceConsumer, base.EnableMetrics) + + if err = presenceConsumer.Start(); err != nil { + logrus.WithError(err).Panicf("failed to start presence consumer") + } userAPIStreamEventProducer := &producers.UserAPIStreamEventProducer{ JetStream: js, @@ -131,15 +140,6 @@ func AddPublicRoutes( logrus.WithError(err).Panicf("failed to start receipts consumer") } - presenceConsumer := consumers.NewPresenceConsumer( - base.ProcessContext, cfg, js, natsClient, syncDB, - notifier, streams.PresenceStreamProvider, - userAPI, - ) - if err = presenceConsumer.Start(); err != nil { - logrus.WithError(err).Panicf("failed to start presence consumer") - } - routing.Setup( base.PublicClientAPIMux, requestPool, syncDB, userAPI, rsAPI, cfg, base.Caches, diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go new file mode 100644 index 000000000..3ce7c64b7 --- /dev/null +++ b/syncapi/syncapi_test.go @@ -0,0 +1,330 @@ +package syncapi + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + keyapi "github.com/matrix-org/dendrite/keyserver/api" + "github.com/matrix-org/dendrite/roomserver/api" + rsapi "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/setup/base" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/dendrite/test/testrig" + userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/nats-io/nats.go" + "github.com/tidwall/gjson" +) + +type syncRoomserverAPI struct { + rsapi.SyncRoomserverAPI + rooms []*test.Room +} + +func (s *syncRoomserverAPI) QueryLatestEventsAndState(ctx context.Context, req *rsapi.QueryLatestEventsAndStateRequest, res *rsapi.QueryLatestEventsAndStateResponse) error { + var room *test.Room + for _, r := range s.rooms { + if r.ID == req.RoomID { + room = r + break + } + } + if room == nil { + res.RoomExists = false + return nil + } + res.RoomVersion = room.Version + return nil // TODO: return state +} + +func (s *syncRoomserverAPI) QuerySharedUsers(ctx context.Context, req *rsapi.QuerySharedUsersRequest, res *rsapi.QuerySharedUsersResponse) error { + res.UserIDsToCount = make(map[string]int) + return nil +} +func (s *syncRoomserverAPI) QueryBulkStateContent(ctx context.Context, req *rsapi.QueryBulkStateContentRequest, res *rsapi.QueryBulkStateContentResponse) error { + return nil +} + +type syncUserAPI struct { + userapi.SyncUserAPI + accounts []userapi.Device +} + +func (s *syncUserAPI) QueryAccessToken(ctx context.Context, req *userapi.QueryAccessTokenRequest, res *userapi.QueryAccessTokenResponse) error { + for _, acc := range s.accounts { + if acc.AccessToken == req.AccessToken { + res.Device = &acc + return nil + } + } + res.Err = "unknown user" + return nil +} + +func (s *syncUserAPI) PerformLastSeenUpdate(ctx context.Context, req *userapi.PerformLastSeenUpdateRequest, res *userapi.PerformLastSeenUpdateResponse) error { + return nil +} + +type syncKeyAPI struct { + keyapi.SyncKeyAPI +} + +func (s *syncKeyAPI) QueryKeyChanges(ctx context.Context, req *keyapi.QueryKeyChangesRequest, res *keyapi.QueryKeyChangesResponse) { +} +func (s *syncKeyAPI) QueryOneTimeKeys(ctx context.Context, req *keyapi.QueryOneTimeKeysRequest, res *keyapi.QueryOneTimeKeysResponse) { + +} + +func TestSyncAPIAccessTokens(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + testSyncAccessTokens(t, dbType) + }) +} + +func testSyncAccessTokens(t *testing.T, dbType test.DBType) { + user := test.NewUser(t) + room := test.NewRoom(t, user) + alice := userapi.Device{ + ID: "ALICEID", + UserID: user.ID, + AccessToken: "ALICE_BEARER_TOKEN", + DisplayName: "Alice", + AccountType: userapi.AccountTypeUser, + } + + base, close := testrig.CreateBaseDendrite(t, dbType) + defer close() + + jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) + defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream) + msgs := toNATSMsgs(t, base, room.Events()) + AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, &syncKeyAPI{}) + testrig.MustPublishMsgs(t, jsctx, msgs...) + + testCases := []struct { + name string + req *http.Request + wantCode int + wantJoinedRooms []string + }{ + { + name: "missing access token", + req: test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ + "timeout": "0", + })), + wantCode: 401, + }, + { + name: "unknown access token", + req: test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ + "access_token": "foo", + "timeout": "0", + })), + wantCode: 401, + }, + { + name: "valid access token", + req: test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ + "access_token": alice.AccessToken, + "timeout": "0", + })), + wantCode: 200, + wantJoinedRooms: []string{room.ID}, + }, + } + // TODO: find a better way + time.Sleep(500 * time.Millisecond) + + for _, tc := range testCases { + w := httptest.NewRecorder() + base.PublicClientAPIMux.ServeHTTP(w, tc.req) + if w.Code != tc.wantCode { + t.Fatalf("%s: got HTTP %d want %d", tc.name, w.Code, tc.wantCode) + } + if tc.wantJoinedRooms != nil { + var res types.Response + if err := json.NewDecoder(w.Body).Decode(&res); err != nil { + t.Fatalf("%s: failed to decode response body: %s", tc.name, err) + } + if len(res.Rooms.Join) != len(tc.wantJoinedRooms) { + t.Errorf("%s: got %v joined rooms, want %v.\nResponse: %+v", tc.name, len(res.Rooms.Join), len(tc.wantJoinedRooms), res) + } + t.Logf("res: %+v", res.Rooms.Join[room.ID]) + + gotEventIDs := make([]string, len(res.Rooms.Join[room.ID].Timeline.Events)) + for i, ev := range res.Rooms.Join[room.ID].Timeline.Events { + gotEventIDs[i] = ev.EventID + } + test.AssertEventIDsEqual(t, gotEventIDs, room.Events()) + } + } +} + +// Tests what happens when we create a room and then /sync before all events from /createRoom have +// been sent to the syncapi +func TestSyncAPICreateRoomSyncEarly(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + testSyncAPICreateRoomSyncEarly(t, dbType) + }) +} + +func testSyncAPICreateRoomSyncEarly(t *testing.T, dbType test.DBType) { + user := test.NewUser(t) + room := test.NewRoom(t, user) + alice := userapi.Device{ + ID: "ALICEID", + UserID: user.ID, + AccessToken: "ALICE_BEARER_TOKEN", + DisplayName: "Alice", + AccountType: userapi.AccountTypeUser, + } + + base, close := testrig.CreateBaseDendrite(t, dbType) + defer close() + + jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) + defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream) + // order is: + // m.room.create + // m.room.member + // m.room.power_levels + // m.room.join_rules + // m.room.history_visibility + msgs := toNATSMsgs(t, base, room.Events()) + sinceTokens := make([]string, len(msgs)) + AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, &syncKeyAPI{}) + for i, msg := range msgs { + testrig.MustPublishMsgs(t, jsctx, msg) + time.Sleep(100 * time.Millisecond) + w := httptest.NewRecorder() + base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ + "access_token": alice.AccessToken, + "timeout": "0", + }))) + if w.Code != 200 { + t.Errorf("got HTTP %d want 200", w.Code) + continue + } + var res types.Response + if err := json.NewDecoder(w.Body).Decode(&res); err != nil { + t.Errorf("failed to decode response body: %s", err) + } + sinceTokens[i] = res.NextBatch.String() + if i == 0 { // create event does not produce a room section + if len(res.Rooms.Join) != 0 { + t.Fatalf("i=%v got %d joined rooms, want 0", i, len(res.Rooms.Join)) + } + } else { // we should have that room somewhere + if len(res.Rooms.Join) != 1 { + t.Fatalf("i=%v got %d joined rooms, want 1", i, len(res.Rooms.Join)) + } + } + } + + // sync with no token "" and with the penultimate token and this should neatly return room events in the timeline block + sinceTokens = append([]string{""}, sinceTokens[:len(sinceTokens)-1]...) + + t.Logf("waited for events to be consumed; syncing with %v", sinceTokens) + for i, since := range sinceTokens { + w := httptest.NewRecorder() + base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ + "access_token": alice.AccessToken, + "timeout": "0", + "since": since, + }))) + if w.Code != 200 { + t.Errorf("since=%s got HTTP %d want 200", since, w.Code) + } + var res types.Response + if err := json.NewDecoder(w.Body).Decode(&res); err != nil { + t.Errorf("failed to decode response body: %s", err) + } + if len(res.Rooms.Join) != 1 { + t.Fatalf("since=%s got %d joined rooms, want 1", since, len(res.Rooms.Join)) + } + t.Logf("since=%s res state:%+v res timeline:%+v", since, res.Rooms.Join[room.ID].State.Events, res.Rooms.Join[room.ID].Timeline.Events) + gotEventIDs := make([]string, len(res.Rooms.Join[room.ID].Timeline.Events)) + for j, ev := range res.Rooms.Join[room.ID].Timeline.Events { + gotEventIDs[j] = ev.EventID + } + test.AssertEventIDsEqual(t, gotEventIDs, room.Events()[i:]) + } +} + +// Test that if we hit /sync we get back presence: online, regardless of whether messages get delivered +// via NATS. Regression test for a flakey test "User sees their own presence in a sync" +func TestSyncAPIUpdatePresenceImmediately(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + testSyncAPIUpdatePresenceImmediately(t, dbType) + }) +} + +func testSyncAPIUpdatePresenceImmediately(t *testing.T, dbType test.DBType) { + user := test.NewUser(t) + alice := userapi.Device{ + ID: "ALICEID", + UserID: user.ID, + AccessToken: "ALICE_BEARER_TOKEN", + DisplayName: "Alice", + AccountType: userapi.AccountTypeUser, + } + + base, close := testrig.CreateBaseDendrite(t, dbType) + base.Cfg.Global.Presence.EnableOutbound = true + base.Cfg.Global.Presence.EnableInbound = true + defer close() + + jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) + defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream) + AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{}, &syncKeyAPI{}) + w := httptest.NewRecorder() + base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ + "access_token": alice.AccessToken, + "timeout": "0", + "set_presence": "online", + }))) + if w.Code != 200 { + t.Fatalf("got HTTP %d want %d", w.Code, 200) + } + var res types.Response + if err := json.NewDecoder(w.Body).Decode(&res); err != nil { + t.Errorf("failed to decode response body: %s", err) + } + if len(res.Presence.Events) != 1 { + t.Fatalf("expected 1 presence events, got: %+v", res.Presence.Events) + } + if res.Presence.Events[0].Sender != alice.UserID { + t.Errorf("sender: got %v want %v", res.Presence.Events[0].Sender, alice.UserID) + } + if res.Presence.Events[0].Type != "m.presence" { + t.Errorf("type: got %v want %v", res.Presence.Events[0].Type, "m.presence") + } + if gjson.ParseBytes(res.Presence.Events[0].Content).Get("presence").Str != "online" { + t.Errorf("content: not online, got %v", res.Presence.Events[0].Content) + } + +} + +func toNATSMsgs(t *testing.T, base *base.BaseDendrite, input []*gomatrixserverlib.HeaderedEvent) []*nats.Msg { + result := make([]*nats.Msg, len(input)) + for i, ev := range input { + var addsStateIDs []string + if ev.StateKey() != nil { + addsStateIDs = append(addsStateIDs, ev.EventID()) + } + result[i] = testrig.NewOutputEventMsg(t, base, ev.RoomID(), api.OutputEvent{ + Type: rsapi.OutputTypeNewRoomEvent, + NewRoomEvent: &rsapi.OutputNewRoomEvent{ + Event: ev, + AddsStateEventIDs: addsStateIDs, + }, + }) + } + return result +} diff --git a/syncapi/types/types.go b/syncapi/types/types.go index ba6b4f8cd..159fa08b6 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -350,6 +350,19 @@ type Response struct { DeviceListsOTKCount map[string]int `json:"device_one_time_keys_count,omitempty"` } +func (r *Response) HasUpdates() bool { + // purposefully exclude DeviceListsOTKCount as we always include them + return (len(r.AccountData.Events) > 0 || + len(r.Presence.Events) > 0 || + len(r.Rooms.Invite) > 0 || + len(r.Rooms.Join) > 0 || + len(r.Rooms.Leave) > 0 || + len(r.Rooms.Peek) > 0 || + len(r.ToDevice.Events) > 0 || + len(r.DeviceLists.Changed) > 0 || + len(r.DeviceLists.Left) > 0) +} + // NewResponse creates an empty response with initialised maps. func NewResponse() *Response { res := Response{} diff --git a/test/base.go b/test/base.go deleted file mode 100644 index 32fc8dc53..000000000 --- a/test/base.go +++ /dev/null @@ -1,18 +0,0 @@ -package test - -import ( - "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" - "github.com/nats-io/nats.go" -) - -func Base(cfg *config.Dendrite) (*base.BaseDendrite, nats.JetStreamContext, *nats.Conn) { - if cfg == nil { - cfg = &config.Dendrite{} - cfg.Defaults(true) - } - cfg.Global.JetStream.InMemory = true - base := base.NewBaseDendrite(cfg, "Tests") - js, jc := base.NATS.Prepare(base.ProcessContext, &cfg.Global.JetStream) - return base, js, jc -} diff --git a/test/db.go b/test/db.go index a1754cd08..c7cb919f6 100644 --- a/test/db.go +++ b/test/db.go @@ -44,8 +44,9 @@ func fatalError(t *testing.T, format string, args ...interface{}) { } func createLocalDB(t *testing.T, dbName string) { - if !Quiet { - t.Log("Note: tests require a postgres install accessible to the current user") + if _, err := exec.LookPath("createdb"); err != nil { + fatalError(t, "Note: tests require a postgres install accessible to the current user") + return } createDB := exec.Command("createdb", dbName) if !Quiet { @@ -63,6 +64,9 @@ func createRemoteDB(t *testing.T, dbName, user, connStr string) { if err != nil { fatalError(t, "failed to open postgres conn with connstr=%s : %s", connStr, err) } + if err = db.Ping(); err != nil { + fatalError(t, "failed to open postgres conn with connstr=%s : %s", connStr, err) + } _, err = db.Exec(fmt.Sprintf(`CREATE DATABASE %s;`, dbName)) if err != nil { pqErr, ok := err.(*pq.Error) diff --git a/test/event.go b/test/event.go index b2e2805ba..73fc656bd 100644 --- a/test/event.go +++ b/test/event.go @@ -52,6 +52,24 @@ func WithUnsigned(unsigned interface{}) eventModifier { } } +func WithKeyID(keyID gomatrixserverlib.KeyID) eventModifier { + return func(e *eventMods) { + e.keyID = keyID + } +} + +func WithPrivateKey(pkey ed25519.PrivateKey) eventModifier { + return func(e *eventMods) { + e.privKey = pkey + } +} + +func WithOrigin(origin gomatrixserverlib.ServerName) eventModifier { + return func(e *eventMods) { + e.origin = origin + } +} + // Reverse a list of events func Reversed(in []*gomatrixserverlib.HeaderedEvent) []*gomatrixserverlib.HeaderedEvent { out := make([]*gomatrixserverlib.HeaderedEvent, len(in)) @@ -64,7 +82,8 @@ func Reversed(in []*gomatrixserverlib.HeaderedEvent) []*gomatrixserverlib.Header func AssertEventIDsEqual(t *testing.T, gotEventIDs []string, wants []*gomatrixserverlib.HeaderedEvent) { t.Helper() if len(gotEventIDs) != len(wants) { - t.Fatalf("length mismatch: got %d events, want %d", len(gotEventIDs), len(wants)) + t.Errorf("length mismatch: got %d events, want %d", len(gotEventIDs), len(wants)) + return } for i := range wants { w := wants[i].EventID() diff --git a/test/http.go b/test/http.go new file mode 100644 index 000000000..37b3648f8 --- /dev/null +++ b/test/http.go @@ -0,0 +1,92 @@ +package test + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "path/filepath" + "sync" + "testing" +) + +type HTTPRequestOpt func(req *http.Request) + +func WithJSONBody(t *testing.T, body interface{}) HTTPRequestOpt { + t.Helper() + b, err := json.Marshal(body) + if err != nil { + t.Fatalf("WithJSONBody: %s", err) + } + return func(req *http.Request) { + req.Body = io.NopCloser(bytes.NewBuffer(b)) + } +} + +func WithQueryParams(qps map[string]string) HTTPRequestOpt { + var vals url.Values = map[string][]string{} + for k, v := range qps { + vals.Set(k, v) + } + return func(req *http.Request) { + req.URL.RawQuery = vals.Encode() + } +} + +func NewRequest(t *testing.T, method, path string, opts ...HTTPRequestOpt) *http.Request { + t.Helper() + req, err := http.NewRequest(method, "http://localhost"+path, nil) + if err != nil { + t.Fatalf("failed to make new HTTP request %v %v : %v", method, path, err) + } + for _, o := range opts { + o(req) + } + return req +} + +// ListenAndServe will listen on a random high-numbered port and attach the given router. +// Returns the base URL to send requests to. Call `cancel` to shutdown the server, which will block until it has closed. +func ListenAndServe(t *testing.T, router http.Handler, withTLS bool) (apiURL string, cancel func()) { + listener, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatalf("failed to listen: %s", err) + } + port := listener.Addr().(*net.TCPAddr).Port + srv := http.Server{} + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + srv.Handler = router + var err error + if withTLS { + certFile := filepath.Join(t.TempDir(), "dendrite.cert") + keyFile := filepath.Join(t.TempDir(), "dendrite.key") + err = NewTLSKey(keyFile, certFile) + if err != nil { + t.Errorf("failed to make TLS key: %s", err) + return + } + err = srv.ServeTLS(listener, certFile, keyFile) + } else { + err = srv.Serve(listener) + } + if err != nil && err != http.ErrServerClosed { + t.Logf("Listen failed: %s", err) + } + }() + s := "" + if withTLS { + s = "s" + } + return fmt.Sprintf("http%s://localhost:%d", s, port), func() { + _ = srv.Shutdown(context.Background()) + wg.Wait() + } +} diff --git a/internal/test/keyring.go b/test/keyring.go similarity index 100% rename from internal/test/keyring.go rename to test/keyring.go diff --git a/internal/test/config.go b/test/keys.go similarity index 61% rename from internal/test/config.go rename to test/keys.go index d8e0c4531..75e3800e0 100644 --- a/internal/test/config.go +++ b/test/keys.go @@ -25,103 +25,19 @@ import ( "io/ioutil" "math/big" "os" - "path/filepath" "strings" "time" - - "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/gomatrixserverlib" - "gopkg.in/yaml.v2" ) const ( - // ConfigFile is the name of the config file for a server. - ConfigFile = "dendrite.yaml" // ServerKeyFile is the name of the file holding the matrix server private key. ServerKeyFile = "server_key.pem" // TLSCertFile is the name of the file holding the TLS certificate used for federation. TLSCertFile = "tls_cert.pem" // TLSKeyFile is the name of the file holding the TLS key used for federation. TLSKeyFile = "tls_key.pem" - // MediaDir is the name of the directory used to store media. - MediaDir = "media" ) -// MakeConfig makes a config suitable for running integration tests. -// Generates new matrix and TLS keys for the server. -func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*config.Dendrite, int, error) { - var cfg config.Dendrite - cfg.Defaults(true) - - port := startPort - assignAddress := func() config.HTTPAddress { - result := config.HTTPAddress(fmt.Sprintf("http://%s:%d", host, port)) - port++ - return result - } - - serverKeyPath := filepath.Join(configDir, ServerKeyFile) - tlsCertPath := filepath.Join(configDir, TLSKeyFile) - tlsKeyPath := filepath.Join(configDir, TLSCertFile) - mediaBasePath := filepath.Join(configDir, MediaDir) - - if err := NewMatrixKey(serverKeyPath); err != nil { - return nil, 0, err - } - - if err := NewTLSKey(tlsKeyPath, tlsCertPath); err != nil { - return nil, 0, err - } - - cfg.Version = config.Version - - cfg.Global.ServerName = gomatrixserverlib.ServerName(assignAddress()) - cfg.Global.PrivateKeyPath = config.Path(serverKeyPath) - - cfg.MediaAPI.BasePath = config.Path(mediaBasePath) - - cfg.Global.JetStream.Addresses = []string{kafkaURI} - - // TODO: Use different databases for the different schemas. - // Using the same database for every schema currently works because - // the table names are globally unique. But we might not want to - // rely on that in the future. - cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(database) - cfg.FederationAPI.Database.ConnectionString = config.DataSource(database) - cfg.KeyServer.Database.ConnectionString = config.DataSource(database) - cfg.MediaAPI.Database.ConnectionString = config.DataSource(database) - cfg.RoomServer.Database.ConnectionString = config.DataSource(database) - cfg.SyncAPI.Database.ConnectionString = config.DataSource(database) - cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(database) - - cfg.AppServiceAPI.InternalAPI.Listen = assignAddress() - cfg.FederationAPI.InternalAPI.Listen = assignAddress() - cfg.KeyServer.InternalAPI.Listen = assignAddress() - cfg.MediaAPI.InternalAPI.Listen = assignAddress() - cfg.RoomServer.InternalAPI.Listen = assignAddress() - cfg.SyncAPI.InternalAPI.Listen = assignAddress() - cfg.UserAPI.InternalAPI.Listen = assignAddress() - - cfg.AppServiceAPI.InternalAPI.Connect = cfg.AppServiceAPI.InternalAPI.Listen - cfg.FederationAPI.InternalAPI.Connect = cfg.FederationAPI.InternalAPI.Listen - cfg.KeyServer.InternalAPI.Connect = cfg.KeyServer.InternalAPI.Listen - cfg.MediaAPI.InternalAPI.Connect = cfg.MediaAPI.InternalAPI.Listen - cfg.RoomServer.InternalAPI.Connect = cfg.RoomServer.InternalAPI.Listen - cfg.SyncAPI.InternalAPI.Connect = cfg.SyncAPI.InternalAPI.Listen - cfg.UserAPI.InternalAPI.Connect = cfg.UserAPI.InternalAPI.Listen - - return &cfg, port, nil -} - -// WriteConfig writes the config file to the directory. -func WriteConfig(cfg *config.Dendrite, configDir string) error { - data, err := yaml.Marshal(cfg) - if err != nil { - return err - } - return ioutil.WriteFile(filepath.Join(configDir, ConfigFile), data, 0666) -} - // NewMatrixKey generates a new ed25519 matrix server key and writes it to a file. func NewMatrixKey(matrixKeyPath string) (err error) { var data [35]byte diff --git a/test/room.go b/test/room.go index 619cb5c9a..6ae403b3f 100644 --- a/test/room.go +++ b/test/room.go @@ -15,7 +15,6 @@ package test import ( - "crypto/ed25519" "encoding/json" "fmt" "sync/atomic" @@ -35,12 +34,6 @@ var ( PresetTrustedPrivateChat Preset = 3 roomIDCounter = int64(0) - - testKeyID = gomatrixserverlib.KeyID("ed25519:test") - testPrivateKey = ed25519.NewKeyFromSeed([]byte{ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, - 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, - }) ) type Room struct { @@ -49,22 +42,25 @@ type Room struct { preset Preset creator *User - authEvents gomatrixserverlib.AuthEvents - events []*gomatrixserverlib.HeaderedEvent + authEvents gomatrixserverlib.AuthEvents + currentState map[string]*gomatrixserverlib.HeaderedEvent + events []*gomatrixserverlib.HeaderedEvent } // Create a new test room. Automatically creates the initial create events. func NewRoom(t *testing.T, creator *User, modifiers ...roomModifier) *Room { t.Helper() counter := atomic.AddInt64(&roomIDCounter, 1) - - // set defaults then let roomModifiers override + if creator.srvName == "" { + t.Fatalf("NewRoom: creator doesn't belong to a server: %+v", *creator) + } r := &Room{ - ID: fmt.Sprintf("!%d:localhost", counter), - creator: creator, - authEvents: gomatrixserverlib.NewAuthEvents(nil), - preset: PresetPublicChat, - Version: gomatrixserverlib.RoomVersionV9, + ID: fmt.Sprintf("!%d:%s", counter, creator.srvName), + creator: creator, + authEvents: gomatrixserverlib.NewAuthEvents(nil), + preset: PresetPublicChat, + Version: gomatrixserverlib.RoomVersionV9, + currentState: make(map[string]*gomatrixserverlib.HeaderedEvent), } for _, m := range modifiers { m(t, r) @@ -73,6 +69,24 @@ func NewRoom(t *testing.T, creator *User, modifiers ...roomModifier) *Room { return r } +func (r *Room) MustGetAuthEventRefsForEvent(t *testing.T, needed gomatrixserverlib.StateNeeded) []gomatrixserverlib.EventReference { + t.Helper() + a, err := needed.AuthEventReferences(&r.authEvents) + if err != nil { + t.Fatalf("MustGetAuthEvents: %v", err) + } + return a +} + +func (r *Room) ForwardExtremities() []string { + if len(r.events) == 0 { + return nil + } + return []string{ + r.events[len(r.events)-1].EventID(), + } +} + func (r *Room) insertCreateEvents(t *testing.T) { t.Helper() var joinRule gomatrixserverlib.JoinRuleContent @@ -88,6 +102,7 @@ func (r *Room) insertCreateEvents(t *testing.T) { joinRule.JoinRule = "public" hisVis.HistoryVisibility = "shared" } + r.CreateAndInsert(t, r.creator, gomatrixserverlib.MRoomCreate, map[string]interface{}{ "creator": r.creator.ID, "room_version": r.Version, @@ -112,16 +127,16 @@ func (r *Room) CreateEvent(t *testing.T, creator *User, eventType string, conten } if mod.privKey == nil { - mod.privKey = testPrivateKey + mod.privKey = creator.privKey } if mod.keyID == "" { - mod.keyID = testKeyID + mod.keyID = creator.keyID } if mod.originServerTS.IsZero() { mod.originServerTS = time.Now() } if mod.origin == "" { - mod.origin = gomatrixserverlib.ServerName("localhost") + mod.origin = creator.srvName } var unsigned gomatrixserverlib.RawJSON @@ -174,13 +189,14 @@ func (r *Room) CreateEvent(t *testing.T, creator *User, eventType string, conten // Add a new event to this room DAG. Not thread-safe. func (r *Room) InsertEvent(t *testing.T, he *gomatrixserverlib.HeaderedEvent) { t.Helper() - // Add the event to the list of auth events + // Add the event to the list of auth/state events r.events = append(r.events, he) if he.StateKey() != nil { err := r.authEvents.AddEvent(he.Unwrap()) if err != nil { t.Fatalf("InsertEvent: failed to add event to auth events: %s", err) } + r.currentState[he.Type()+" "+*he.StateKey()] = he } } @@ -188,6 +204,16 @@ func (r *Room) Events() []*gomatrixserverlib.HeaderedEvent { return r.events } +func (r *Room) CurrentState() []*gomatrixserverlib.HeaderedEvent { + events := make([]*gomatrixserverlib.HeaderedEvent, len(r.currentState)) + i := 0 + for _, e := range r.currentState { + events[i] = e + i++ + } + return events +} + func (r *Room) CreateAndInsert(t *testing.T, creator *User, eventType string, content interface{}, mods ...eventModifier) *gomatrixserverlib.HeaderedEvent { t.Helper() he := r.CreateEvent(t, creator, eventType, content, mods...) diff --git a/internal/test/slice.go b/test/slice.go similarity index 100% rename from internal/test/slice.go rename to test/slice.go diff --git a/test/testrig/base.go b/test/testrig/base.go new file mode 100644 index 000000000..facb49f3e --- /dev/null +++ b/test/testrig/base.go @@ -0,0 +1,91 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testrig + +import ( + "errors" + "fmt" + "io/fs" + "os" + "strings" + "testing" + + "github.com/matrix-org/dendrite/setup/base" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/test" + "github.com/nats-io/nats.go" +) + +func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, func()) { + var cfg config.Dendrite + cfg.Defaults(false) + cfg.Global.JetStream.InMemory = true + + switch dbType { + case test.DBTypePostgres: + cfg.Global.Defaults(true) // autogen a signing key + cfg.MediaAPI.Defaults(true) // autogen a media path + // use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use + // the file system event with InMemory=true :( + cfg.Global.JetStream.TopicPrefix = fmt.Sprintf("Test_%d_", dbType) + connStr, close := test.PrepareDBConnectionString(t, dbType) + cfg.Global.DatabaseOptions = config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + MaxOpenConnections: 10, + MaxIdleConnections: 2, + ConnMaxLifetimeSeconds: 60, + } + return base.NewBaseDendrite(&cfg, "Test", base.DisableMetrics), close + case test.DBTypeSQLite: + cfg.Defaults(true) // sets a sqlite db per component + // use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use + // the file system event with InMemory=true :( + cfg.Global.JetStream.TopicPrefix = fmt.Sprintf("Test_%d_", dbType) + return base.NewBaseDendrite(&cfg, "Test", base.DisableMetrics), func() { + // cleanup db files. This risks getting out of sync as we add more database strings :( + dbFiles := []config.DataSource{ + cfg.AppServiceAPI.Database.ConnectionString, + cfg.FederationAPI.Database.ConnectionString, + cfg.KeyServer.Database.ConnectionString, + cfg.MSCs.Database.ConnectionString, + cfg.MediaAPI.Database.ConnectionString, + cfg.RoomServer.Database.ConnectionString, + cfg.SyncAPI.Database.ConnectionString, + cfg.UserAPI.AccountDatabase.ConnectionString, + } + for _, fileURI := range dbFiles { + path := strings.TrimPrefix(string(fileURI), "file:") + err := os.Remove(path) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("failed to cleanup sqlite db '%s': %s", fileURI, err) + } + } + } + default: + t.Fatalf("unknown db type: %v", dbType) + } + return nil, nil +} + +func Base(cfg *config.Dendrite) (*base.BaseDendrite, nats.JetStreamContext, *nats.Conn) { + if cfg == nil { + cfg = &config.Dendrite{} + cfg.Defaults(true) + } + cfg.Global.JetStream.InMemory = true + base := base.NewBaseDendrite(cfg, "Tests") + js, jc := base.NATS.Prepare(base.ProcessContext, &cfg.Global.JetStream) + return base, js, jc +} diff --git a/test/testrig/jetstream.go b/test/testrig/jetstream.go new file mode 100644 index 000000000..74cf95062 --- /dev/null +++ b/test/testrig/jetstream.go @@ -0,0 +1,35 @@ +package testrig + +import ( + "encoding/json" + "testing" + + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/setup/base" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/nats-io/nats.go" +) + +func MustPublishMsgs(t *testing.T, jsctx nats.JetStreamContext, msgs ...*nats.Msg) { + t.Helper() + for _, msg := range msgs { + if _, err := jsctx.PublishMsg(msg); err != nil { + t.Fatalf("MustPublishMsgs: failed to publish message: %s", err) + } + } +} + +func NewOutputEventMsg(t *testing.T, base *base.BaseDendrite, roomID string, update api.OutputEvent) *nats.Msg { + t.Helper() + msg := &nats.Msg{ + Subject: base.Cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent), + Header: nats.Header{}, + } + msg.Header.Set(jetstream.RoomID, roomID) + var err error + msg.Data, err = json.Marshal(update) + if err != nil { + t.Fatalf("failed to marshal update: %s", err) + } + return msg +} diff --git a/test/user.go b/test/user.go index 41a66e1c4..0020098a5 100644 --- a/test/user.go +++ b/test/user.go @@ -15,22 +15,64 @@ package test import ( + "crypto/ed25519" "fmt" "sync/atomic" + "testing" + + "github.com/matrix-org/gomatrixserverlib" ) var ( userIDCounter = int64(0) + + serverName = gomatrixserverlib.ServerName("test") + keyID = gomatrixserverlib.KeyID("ed25519:test") + privateKey = ed25519.NewKeyFromSeed([]byte{ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + }) + + // private keys that tests can use + PrivateKeyA = ed25519.NewKeyFromSeed([]byte{ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 77, + }) + PrivateKeyB = ed25519.NewKeyFromSeed([]byte{ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 66, + }) ) type User struct { ID string + // key ID and private key of the server who has this user, if known. + keyID gomatrixserverlib.KeyID + privKey ed25519.PrivateKey + srvName gomatrixserverlib.ServerName } -func NewUser() *User { - counter := atomic.AddInt64(&userIDCounter, 1) - u := &User{ - ID: fmt.Sprintf("@%d:localhost", counter), +type UserOpt func(*User) + +func WithSigningServer(srvName gomatrixserverlib.ServerName, keyID gomatrixserverlib.KeyID, privKey ed25519.PrivateKey) UserOpt { + return func(u *User) { + u.keyID = keyID + u.privKey = privKey + u.srvName = srvName } - return u +} + +func NewUser(t *testing.T, opts ...UserOpt) *User { + counter := atomic.AddInt64(&userIDCounter, 1) + var u User + for _, opt := range opts { + opt(&u) + } + if u.keyID == "" || u.srvName == "" || u.privKey == nil { + t.Logf("NewUser: missing signing server credentials; using default.") + WithSigningServer(serverName, keyID, privateKey)(&u) + } + u.ID = fmt.Sprintf("@%d:%s", counter, u.srvName) + t.Logf("NewUser: created user %s", u.ID) + return &u } diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go index 35de8d6f7..32583fb0f 100644 --- a/userapi/storage/storage_test.go +++ b/userapi/storage/storage_test.go @@ -43,7 +43,7 @@ func Test_AccountData(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { db, close := mustCreateDatabase(t, dbType) defer close() - alice := test.NewUser() + alice := test.NewUser(t) localpart, _, err := gomatrixserverlib.SplitID('@', alice.ID) assert.NoError(t, err) @@ -74,7 +74,7 @@ func Test_Accounts(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { db, close := mustCreateDatabase(t, dbType) defer close() - alice := test.NewUser() + alice := test.NewUser(t) aliceLocalpart, _, err := gomatrixserverlib.SplitID('@', alice.ID) assert.NoError(t, err) @@ -128,7 +128,7 @@ func Test_Accounts(t *testing.T) { } func Test_Devices(t *testing.T) { - alice := test.NewUser() + alice := test.NewUser(t) localpart, _, err := gomatrixserverlib.SplitID('@', alice.ID) assert.NoError(t, err) deviceID := util.RandomString(8) @@ -212,7 +212,7 @@ func Test_Devices(t *testing.T) { } func Test_KeyBackup(t *testing.T) { - alice := test.NewUser() + alice := test.NewUser(t) room := test.NewRoom(t, alice) test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { @@ -291,7 +291,7 @@ func Test_KeyBackup(t *testing.T) { } func Test_LoginToken(t *testing.T) { - alice := test.NewUser() + alice := test.NewUser(t) test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { db, close := mustCreateDatabase(t, dbType) defer close() @@ -321,7 +321,7 @@ func Test_LoginToken(t *testing.T) { } func Test_OpenID(t *testing.T) { - alice := test.NewUser() + alice := test.NewUser(t) token := util.RandomString(24) test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { @@ -341,7 +341,7 @@ func Test_OpenID(t *testing.T) { } func Test_Profile(t *testing.T) { - alice := test.NewUser() + alice := test.NewUser(t) aliceLocalpart, _, err := gomatrixserverlib.SplitID('@', alice.ID) assert.NoError(t, err) @@ -379,7 +379,7 @@ func Test_Profile(t *testing.T) { } func Test_Pusher(t *testing.T) { - alice := test.NewUser() + alice := test.NewUser(t) aliceLocalpart, _, err := gomatrixserverlib.SplitID('@', alice.ID) assert.NoError(t, err) @@ -430,7 +430,7 @@ func Test_Pusher(t *testing.T) { } func Test_ThreePID(t *testing.T) { - alice := test.NewUser() + alice := test.NewUser(t) aliceLocalpart, _, err := gomatrixserverlib.SplitID('@', alice.ID) assert.NoError(t, err) @@ -467,7 +467,7 @@ func Test_ThreePID(t *testing.T) { } func Test_Notification(t *testing.T) { - alice := test.NewUser() + alice := test.NewUser(t) aliceLocalpart, _, err := gomatrixserverlib.SplitID('@', alice.ID) assert.NoError(t, err) room := test.NewRoom(t, alice) diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go index 7038f5ee8..364744591 100644 --- a/userapi/userapi_test.go +++ b/userapi/userapi_test.go @@ -24,7 +24,6 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/internal/httputil" - internalTest "github.com/matrix-org/dendrite/internal/test" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/dendrite/userapi/inthttp" @@ -135,7 +134,7 @@ func TestQueryProfile(t *testing.T) { t.Run("HTTP API", func(t *testing.T) { router := mux.NewRouter().PathPrefix(httputil.InternalPathPrefix).Subrouter() userapi.AddInternalRoutes(router, userAPI) - apiURL, cancel := internalTest.ListenAndServe(t, router, false) + apiURL, cancel := test.ListenAndServe(t, router, false) defer cancel() httpAPI, err := inthttp.NewUserAPIClient(apiURL, &http.Client{}) if err != nil { diff --git a/userapi/util/phonehomestats.go b/userapi/util/phonehomestats.go index e24daba6b..ad93a50e3 100644 --- a/userapi/util/phonehomestats.go +++ b/userapi/util/phonehomestats.go @@ -39,7 +39,7 @@ type phoneHomeStats struct { cfg *config.Dendrite db storage.Statistics isMonolith bool - client *gomatrixserverlib.Client + client *http.Client } type timestampToRUUsage struct { @@ -55,9 +55,10 @@ func StartPhoneHomeCollector(startTime time.Time, cfg *config.Dendrite, statsDB cfg: cfg, db: statsDB, isMonolith: cfg.IsMonolith, - client: gomatrixserverlib.NewClient( - gomatrixserverlib.WithTimeout(time.Second * 30), - ), + client: &http.Client{ + Timeout: time.Second * 30, + Transport: http.DefaultTransport, + }, } // start initial run after 5min @@ -151,7 +152,8 @@ func (p *phoneHomeStats) collect() { } request.Header.Set("User-Agent", "Dendrite/"+internal.VersionString()) - if _, err = p.client.DoHTTPRequest(ctx, request); err != nil { + _, err = p.client.Do(request) + if err != nil { logrus.WithError(err).Error("unable to send anonymous stats") return }