Merge branch 'main' of github.com:matrix-org/dendrite into s7evink/pocfts

This commit is contained in:
Till Faelligen 2022-05-15 12:23:18 +02:00
commit 1d02a5888f
18 changed files with 119 additions and 66 deletions

View file

@ -1,5 +1,20 @@
# Changelog
## Dendrite 0.8.5 (2022-05-13)
### Features
* New living documentation available at <https://matrix-org.github.io/dendrite/>, including new installation instructions
* The built-in NATS Server has been updated to version 2.8.2
### Fixes
* Monolith deployments will no longer panic at startup if given a config file that does not include the `internal_api` and `external_api` options
* State resolution v2 now correctly identifies other events related to power events, which should fix some event auth issues
* The latest events updater will no longer implicitly trust the new forward extremities when calculating the current room state, which may help to avoid some state resets
* The one-time key count is now correctly returned in `/sync` even if the request otherwise timed out, which should reduce the chance that unnecessary one-time keys will be uploaded by clients
* The `create-account` tool should now work properly when the database is configured using the global connection pool
## Dendrite 0.8.4 (2022-05-10)
### Fixes

View file

@ -53,7 +53,7 @@ The [Federation Tester](https://federationtester.matrix.org) can be used to veri
## Get started
If you wish to build a fully-federating Dendrite instance, see [the Installation documentation](docs/installation). For running in Docker, see [build/docker](build/docker).
If you wish to build a fully-federating Dendrite instance, see [the Installation documentation](https://matrix-org.github.io/dendrite/installation). For running in Docker, see [build/docker](build/docker).
The following instructions are enough to get Dendrite started as a non-federating test deployment using self-signed certificates and SQLite databases:

View file

@ -225,7 +225,7 @@ func (m *DendriteMonolith) Start() {
pk = sk.Public().(ed25519.PublicKey)
}
m.listener, err = net.Listen("tcp", ":65432")
m.listener, err = net.Listen("tcp", "localhost:65432")
if err != nil {
panic(err)
}

4
go.mod
View file

@ -1,6 +1,6 @@
module github.com/matrix-org/dendrite
replace github.com/nats-io/nats-server/v2 => github.com/neilalexander/nats-server/v2 v2.8.1-0.20220419100629-2278c94774f9
replace github.com/nats-io/nats-server/v2 => github.com/neilalexander/nats-server/v2 v2.8.3-0.20220513095553-73a9a246d34f
replace github.com/nats-io/nats.go => github.com/neilalexander/nats.go v1.13.1-0.20220419101051-b262d9f0be1e
@ -31,7 +31,7 @@ require (
github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
github.com/matrix-org/gomatrixserverlib v0.0.0-20220509120958-8d818048c34c
github.com/matrix-org/gomatrixserverlib v0.0.0-20220513103617-eee8fd528433
github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
github.com/mattn/go-sqlite3 v1.14.10

10
go.sum
View file

@ -836,8 +836,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
github.com/matrix-org/gomatrixserverlib v0.0.0-20220509120958-8d818048c34c h1:KqzqFWxvs90pcDaW9QEveW+Q5JcEYuNnKyaqXc+ohno=
github.com/matrix-org/gomatrixserverlib v0.0.0-20220509120958-8d818048c34c/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
github.com/matrix-org/gomatrixserverlib v0.0.0-20220513103617-eee8fd528433 h1:nwAlThHGPI2EAAJklXvgMcdhXF6ZiHp60+fmaYMoaDA=
github.com/matrix-org/gomatrixserverlib v0.0.0-20220513103617-eee8fd528433/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE=
github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
@ -932,8 +932,8 @@ github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uY
github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
github.com/neilalexander/nats-server/v2 v2.8.1-0.20220419100629-2278c94774f9 h1:VGU5HYAwy8LRbSkrT+kCHvujVmwK8Aa/vc1O+eReTbM=
github.com/neilalexander/nats-server/v2 v2.8.1-0.20220419100629-2278c94774f9/go.mod h1:5vic7C58BFEVltiZhs7Kq81q2WcEPhJPsmNv1FOrdv0=
github.com/neilalexander/nats-server/v2 v2.8.3-0.20220513095553-73a9a246d34f h1:Fc+TjdV1mOy0oISSzfoxNWdTqjg7tN/Vdgf+B2cwvdo=
github.com/neilalexander/nats-server/v2 v2.8.3-0.20220513095553-73a9a246d34f/go.mod h1:vIdpKz3OG+DCg4q/xVPdXHoztEyKDWRtykQ4N7hd7C4=
github.com/neilalexander/nats.go v1.13.1-0.20220419101051-b262d9f0be1e h1:kNIzIzj2OvnlreA+sTJ12nWJzTP3OSLNKDL/Iq9mF6Y=
github.com/neilalexander/nats.go v1.13.1-0.20220419101051-b262d9f0be1e/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 h1:lrVQzBtkeQEGGYUHwSX1XPe1E5GL6U3KYCNe2G4bncQ=
@ -1326,7 +1326,7 @@ golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 h1:NvGWuYG8dkDHFSKksI1P9faiVJ9rayE6l0+ouWVIDs8=
golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=

View file

@ -17,7 +17,7 @@ var build string
const (
VersionMajor = 0
VersionMinor = 8
VersionPatch = 4
VersionPatch = 5
VersionTag = "" // example: "rc1"
)

View file

@ -233,12 +233,19 @@ func (u *latestEventsUpdater) latestState() error {
}
}
// Get a list of the current latest events. This may or may not
// include the new event from the input path, depending on whether
// it is a forward extremity or not.
latestStateAtEvents := make([]types.StateAtEvent, len(u.latest))
for i := range u.latest {
latestStateAtEvents[i] = u.latest[i].StateAtEvent
// Take the old set of extremities and the new set of extremities and
// mash them together into a list. This may or may not include the new event
// from the input path, depending on whether it became a forward extremity
// or not. We'll then run state resolution across all of them to determine
// the new current state of the room. Including the old extremities here
// ensures that new forward extremities with bad state snapshots (from
// possible malicious actors) can't completely corrupt the room state
// away from what it was before.
combinedExtremities := types.StateAtEventAndReferences(append(u.oldLatest, u.latest...))
combinedExtremities = combinedExtremities[:util.SortAndUnique(combinedExtremities)]
latestStateAtEvents := make([]types.StateAtEvent, len(combinedExtremities))
for i := range combinedExtremities {
latestStateAtEvents[i] = combinedExtremities[i].StateAtEvent
}
// Takes the NIDs of the latest events and creates a state snapshot

View file

@ -18,6 +18,7 @@ package types
import (
"encoding/json"
"sort"
"strings"
"github.com/matrix-org/gomatrixserverlib"
"golang.org/x/crypto/blake2b"
@ -166,6 +167,20 @@ type StateAtEventAndReference struct {
gomatrixserverlib.EventReference
}
type StateAtEventAndReferences []StateAtEventAndReference
func (s StateAtEventAndReferences) Less(a, b int) bool {
return strings.Compare(s[a].EventID, s[b].EventID) < 0
}
func (s StateAtEventAndReferences) Len() int {
return len(s)
}
func (s StateAtEventAndReferences) Swap(a, b int) {
s[a], s[b] = s[b], s[a]
}
// An Event is a gomatrixserverlib.Event with the numeric event ID attached.
// It is when performing bulk event lookup in the database.
type Event struct {

View file

@ -50,11 +50,14 @@ func (c *AppServiceAPI) Defaults(generate bool) {
}
func (c *AppServiceAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkURL(configErrs, "app_service_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "app_service_api.internal_api.bind", string(c.InternalAPI.Connect))
if c.Matrix.DatabaseOptions.ConnectionString == "" {
checkNotEmpty(configErrs, "app_service_api.database.connection_string", string(c.Database.ConnectionString))
}
if isMonolith { // polylith required configs below
return
}
checkURL(configErrs, "app_service_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "app_service_api.internal_api.connect", string(c.InternalAPI.Connect))
}
// ApplicationServiceNamespace is the namespace that a specific application

View file

@ -67,19 +67,13 @@ func (c *ClientAPI) Defaults(generate bool) {
}
func (c *ClientAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkURL(configErrs, "client_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "client_api.internal_api.connect", string(c.InternalAPI.Connect))
if !isMonolith {
checkURL(configErrs, "client_api.external_api.listen", string(c.ExternalAPI.Listen))
}
if c.RecaptchaEnabled {
checkNotEmpty(configErrs, "client_api.recaptcha_public_key", string(c.RecaptchaPublicKey))
checkNotEmpty(configErrs, "client_api.recaptcha_private_key", string(c.RecaptchaPrivateKey))
checkNotEmpty(configErrs, "client_api.recaptcha_siteverify_api", string(c.RecaptchaSiteVerifyAPI))
}
c.TURN.Verify(configErrs)
c.RateLimiting.Verify(configErrs)
if c.RecaptchaEnabled {
checkNotEmpty(configErrs, "client_api.recaptcha_public_key", c.RecaptchaPublicKey)
checkNotEmpty(configErrs, "client_api.recaptcha_private_key", c.RecaptchaPrivateKey)
checkNotEmpty(configErrs, "client_api.recaptcha_siteverify_api", c.RecaptchaSiteVerifyAPI)
}
// Ensure there is any spam counter measure when enabling registration
if !c.RegistrationDisabled && !c.OpenRegistrationWithoutVerificationEnabled {
if !c.RecaptchaEnabled {
@ -93,6 +87,12 @@ func (c *ClientAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
)
}
}
if isMonolith { // polylith required configs below
return
}
checkURL(configErrs, "client_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "client_api.internal_api.connect", string(c.InternalAPI.Connect))
checkURL(configErrs, "client_api.external_api.listen", string(c.ExternalAPI.Listen))
}
type TURN struct {

View file

@ -34,24 +34,24 @@ func (c *FederationAPI) Defaults(generate bool) {
c.InternalAPI.Listen = "http://localhost:7772"
c.InternalAPI.Connect = "http://localhost:7772"
c.ExternalAPI.Listen = "http://[::]:8072"
c.FederationMaxRetries = 16
c.DisableTLSValidation = false
c.Database.Defaults(10)
if generate {
c.Database.ConnectionString = "file:federationapi.db"
}
c.FederationMaxRetries = 16
c.DisableTLSValidation = false
}
func (c *FederationAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkURL(configErrs, "federation_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "federation_api.internal_api.connect", string(c.InternalAPI.Connect))
if !isMonolith {
checkURL(configErrs, "federation_api.external_api.listen", string(c.ExternalAPI.Listen))
}
if c.Matrix.DatabaseOptions.ConnectionString == "" {
checkNotEmpty(configErrs, "federation_api.database.connection_string", string(c.Database.ConnectionString))
}
if isMonolith { // polylith required configs below
return
}
checkURL(configErrs, "federation_api.external_api.listen", string(c.ExternalAPI.Listen))
checkURL(configErrs, "federation_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "federation_api.internal_api.connect", string(c.InternalAPI.Connect))
}
// The config for setting a proxy to use for server->server requests

View file

@ -36,9 +36,10 @@ func (c *JetStream) Defaults(generate bool) {
}
func (c *JetStream) Verify(configErrs *ConfigErrors, isMonolith bool) {
if isMonolith { // polylith required configs below
return
}
// If we are running in a polylith deployment then we need at least
// one NATS JetStream server to talk to.
if !isMonolith {
checkNotZero(configErrs, "global.jetstream.addresses", int64(len(c.Addresses)))
}
checkNotZero(configErrs, "global.jetstream.addresses", int64(len(c.Addresses)))
}

View file

@ -18,9 +18,12 @@ func (c *KeyServer) Defaults(generate bool) {
}
func (c *KeyServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkURL(configErrs, "key_server.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "key_server.internal_api.bind", string(c.InternalAPI.Connect))
if c.Matrix.DatabaseOptions.ConnectionString == "" {
checkNotEmpty(configErrs, "key_server.database.connection_string", string(c.Database.ConnectionString))
}
if isMonolith { // polylith required configs below
return
}
checkURL(configErrs, "key_server.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "key_server.internal_api.connect", string(c.InternalAPI.Connect))
}

View file

@ -42,26 +42,19 @@ func (c *MediaAPI) Defaults(generate bool) {
c.InternalAPI.Listen = "http://localhost:7774"
c.InternalAPI.Connect = "http://localhost:7774"
c.ExternalAPI.Listen = "http://[::]:8074"
c.MaxFileSizeBytes = DefaultMaxFileSizeBytes
c.MaxThumbnailGenerators = 10
c.Database.Defaults(5)
if generate {
c.Database.ConnectionString = "file:mediaapi.db"
c.BasePath = "./media_store"
}
c.MaxFileSizeBytes = DefaultMaxFileSizeBytes
c.MaxThumbnailGenerators = 10
}
func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkURL(configErrs, "media_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "media_api.internal_api.connect", string(c.InternalAPI.Connect))
if !isMonolith {
checkURL(configErrs, "media_api.external_api.listen", string(c.ExternalAPI.Listen))
}
if c.Matrix.DatabaseOptions.ConnectionString == "" {
checkNotEmpty(configErrs, "media_api.database.connection_string", string(c.Database.ConnectionString))
}
checkNotEmpty(configErrs, "media_api.base_path", string(c.BasePath))
checkPositive(configErrs, "media_api.max_file_size_bytes", int64(c.MaxFileSizeBytes))
checkPositive(configErrs, "media_api.max_thumbnail_generators", int64(c.MaxThumbnailGenerators))
@ -70,4 +63,10 @@ func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkPositive(configErrs, fmt.Sprintf("media_api.thumbnail_sizes[%d].width", i), int64(size.Width))
checkPositive(configErrs, fmt.Sprintf("media_api.thumbnail_sizes[%d].height", i), int64(size.Height))
}
if isMonolith { // polylith required configs below
return
}
checkURL(configErrs, "media_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "media_api.internal_api.connect", string(c.InternalAPI.Connect))
checkURL(configErrs, "media_api.external_api.listen", string(c.ExternalAPI.Listen))
}

View file

@ -18,9 +18,12 @@ func (c *RoomServer) Defaults(generate bool) {
}
func (c *RoomServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkURL(configErrs, "room_server.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "room_server.internal_ap.bind", string(c.InternalAPI.Connect))
if c.Matrix.DatabaseOptions.ConnectionString == "" {
checkNotEmpty(configErrs, "room_server.database.connection_string", string(c.Database.ConnectionString))
}
if isMonolith { // polylith required configs below
return
}
checkURL(configErrs, "room_server.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "room_server.internal_ap.connect", string(c.InternalAPI.Connect))
}

View file

@ -22,12 +22,13 @@ func (c *SyncAPI) Defaults(generate bool) {
}
func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkURL(configErrs, "sync_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "sync_api.internal_api.bind", string(c.InternalAPI.Connect))
if !isMonolith {
checkURL(configErrs, "sync_api.external_api.listen", string(c.ExternalAPI.Listen))
}
if c.Matrix.DatabaseOptions.ConnectionString == "" {
checkNotEmpty(configErrs, "sync_api.database", string(c.Database.ConnectionString))
}
if isMonolith { // polylith required configs below
return
}
checkURL(configErrs, "sync_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "sync_api.internal_api.connect", string(c.InternalAPI.Connect))
checkURL(configErrs, "sync_api.external_api.listen", string(c.ExternalAPI.Listen))
}

View file

@ -26,19 +26,22 @@ const DefaultOpenIDTokenLifetimeMS = 3600000 // 60 minutes
func (c *UserAPI) Defaults(generate bool) {
c.InternalAPI.Listen = "http://localhost:7781"
c.InternalAPI.Connect = "http://localhost:7781"
c.BCryptCost = bcrypt.DefaultCost
c.OpenIDTokenLifetimeMS = DefaultOpenIDTokenLifetimeMS
c.AccountDatabase.Defaults(10)
if generate {
c.AccountDatabase.ConnectionString = "file:userapi_accounts.db"
}
c.BCryptCost = bcrypt.DefaultCost
c.OpenIDTokenLifetimeMS = DefaultOpenIDTokenLifetimeMS
}
func (c *UserAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkURL(configErrs, "user_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "user_api.internal_api.connect", string(c.InternalAPI.Connect))
checkPositive(configErrs, "user_api.openid_token_lifetime_ms", c.OpenIDTokenLifetimeMS)
if c.Matrix.DatabaseOptions.ConnectionString == "" {
checkNotEmpty(configErrs, "user_api.account_database.connection_string", string(c.AccountDatabase.ConnectionString))
}
checkPositive(configErrs, "user_api.openid_token_lifetime_ms", c.OpenIDTokenLifetimeMS)
if isMonolith { // polylith required configs below
return
}
checkURL(configErrs, "user_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "user_api.internal_api.connect", string(c.InternalAPI.Connect))
}

View file

@ -253,9 +253,12 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
// We should always try to include OTKs in sync responses, otherwise clients might upload keys
// even if that's not required. See also:
// https://github.com/matrix-org/synapse/blob/29f06704b8871a44926f7c99e73cf4a978fb8e81/synapse/rest/client/sync.py#L276-L281
err = internal.DeviceOTKCounts(syncReq.Context, rp.keyAPI, syncReq.Device.UserID, syncReq.Device.ID, syncReq.Response)
if err != nil {
syncReq.Log.WithError(err).Error("failed to get OTK counts")
// Only try to get OTKs if the context isn't already done.
if syncReq.Context.Err() == nil {
err = internal.DeviceOTKCounts(syncReq.Context, rp.keyAPI, syncReq.Device.UserID, syncReq.Device.ID, syncReq.Response)
if err != nil && err != context.Canceled {
syncReq.Log.WithError(err).Warn("failed to get OTK counts")
}
}
return util.JSONResponse{
Code: http.StatusOK,