diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..47b85b3e1 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @matrix-org/dendrite-core \ No newline at end of file diff --git a/appservice/appservice.go b/appservice/appservice.go index fd903b980..5f16c10b3 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -131,10 +131,11 @@ func generateAppServiceAccount( } var devRes userapi.PerformDeviceCreationResponse err = userAPI.PerformDeviceCreation(context.Background(), &userapi.PerformDeviceCreationRequest{ - Localpart: as.SenderLocalpart, - AccessToken: as.ASToken, - DeviceID: &as.SenderLocalpart, - DeviceDisplayName: &as.SenderLocalpart, + Localpart: as.SenderLocalpart, + AccessToken: as.ASToken, + DeviceID: &as.SenderLocalpart, + DeviceDisplayName: &as.SenderLocalpart, + NoDeviceListUpdate: true, }, &devRes) return err } diff --git a/are-we-synapse-yet.py b/are-we-synapse-yet.py index 92c7b82b8..10b1be28a 100755 --- a/are-we-synapse-yet.py +++ b/are-we-synapse-yet.py @@ -177,6 +177,10 @@ def print_stats(header_name, gid_to_tests, gid_to_name, verbose): line = "%s: %s (%d/%d tests)" % (gid_to_name[gid].ljust(25, ' '), pct.rjust(4, ' '), group_passing, group_total) subsections.append(line) subsection_test_names[line] = test_names_and_marks + + # avoid errors when trying to divide by 0 + if total_tests == 0: + return pct = "{0:.0f}%".format(total_passing/total_tests * 100) print("%s: %s (%d/%d tests)" % (header_name, pct, total_passing, total_tests)) diff --git a/build/docker/docker-compose.deps.yml b/build/docker/docker-compose.deps.yml index 0732e1813..aa0651889 100644 --- a/build/docker/docker-compose.deps.yml +++ b/build/docker/docker-compose.deps.yml @@ -3,7 +3,7 @@ services: # PostgreSQL is needed for both polylith and monolith modes. postgres: hostname: postgres - image: postgres:11 + image: postgres:14 restart: always volumes: - ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 95176fff7..416453da4 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -310,7 +310,7 @@ func (m *DendriteMonolith) Start() { rsAPI := roomserver.NewInternalAPI(base) fsAPI := federationapi.NewInternalAPI( - base, federation, rsAPI, m.userAPI, base.Caches, true, + base, federation, rsAPI, m.userAPI, base.Caches, keyRing, true, ) keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI) @@ -325,8 +325,7 @@ func (m *DendriteMonolith) Start() { // The underlying roomserver implementation needs to be able to call the fedsender. // This is different to rsAPI which can be the http client which doesn't need this dependency - rsAPI.SetFederationAPI(fsAPI) - rsAPI.SetKeyring(keyRing) + rsAPI.SetFederationAPI(fsAPI, keyRing) monolith := setup.Monolith{ Config: base.Cfg, diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go index b0b955580..0410c7cd3 100644 --- a/build/gobind-yggdrasil/monolith.go +++ b/build/gobind-yggdrasil/monolith.go @@ -118,7 +118,7 @@ func (m *DendriteMonolith) Start() { keyAPI.SetUserAPI(userAPI) fsAPI := federationapi.NewInternalAPI( - base, federation, rsAPI, userAPI, base.Caches, true, + base, federation, rsAPI, userAPI, base.Caches, keyRing, true, ) eduInputAPI := eduserver.NewInternalAPI( @@ -130,8 +130,7 @@ func (m *DendriteMonolith) Start() { // The underlying roomserver implementation needs to be able to call the fedsender. // This is different to rsAPI which can be the http client which doesn't need this dependency - rsAPI.SetFederationAPI(fsAPI) - rsAPI.SetKeyring(keyRing) + rsAPI.SetFederationAPI(fsAPI, keyRing) monolith := setup.Monolith{ Config: base.Cfg, diff --git a/clientapi/auth/auth.go b/clientapi/auth/auth.go index b4c39ae38..c850bf91e 100644 --- a/clientapi/auth/auth.go +++ b/clientapi/auth/auth.go @@ -70,11 +70,11 @@ func VerifyUserFromRequest( jsonErr := jsonerror.InternalServerError() return nil, &jsonErr } - if res.Err != nil { - if forbidden, ok := res.Err.(*api.ErrorForbidden); ok { + if res.Err != "" { + if strings.HasPrefix(strings.ToLower(res.Err), "forbidden:") { // TODO: use actual error and no string comparison return nil, &util.JSONResponse{ Code: http.StatusForbidden, - JSON: jsonerror.Forbidden(forbidden.Message), + JSON: jsonerror.Forbidden(res.Err), } } } diff --git a/clientapi/auth/password.go b/clientapi/auth/password.go index a66e2fe76..7dd21b3f2 100644 --- a/clientapi/auth/password.go +++ b/clientapi/auth/password.go @@ -17,6 +17,7 @@ package auth import ( "context" "net/http" + "strings" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/userutil" @@ -48,7 +49,8 @@ func (t *LoginTypePassword) Request() interface{} { func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, *util.JSONResponse) { r := req.(*PasswordRequest) - username := r.Username() + // Squash username to all lowercase letters + username := strings.ToLower(r.Username()) if username == "" { return nil, &util.JSONResponse{ Code: http.StatusUnauthorized, diff --git a/clientapi/routing/key_backup.go b/clientapi/routing/key_backup.go index ce62a047a..9d2ff87fd 100644 --- a/clientapi/routing/key_backup.go +++ b/clientapi/routing/key_backup.go @@ -62,12 +62,14 @@ func CreateKeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI, return *resErr } var performKeyBackupResp userapi.PerformKeyBackupResponse - userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{ + if err := userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{ UserID: device.UserID, Version: "", AuthData: kb.AuthData, Algorithm: kb.Algorithm, - }, &performKeyBackupResp) + }, &performKeyBackupResp); err != nil { + return jsonerror.InternalServerError() + } if performKeyBackupResp.Error != "" { if performKeyBackupResp.BadInput { return util.JSONResponse{ @@ -123,12 +125,14 @@ func ModifyKeyBackupVersionAuthData(req *http.Request, userAPI userapi.UserInter return *resErr } var performKeyBackupResp userapi.PerformKeyBackupResponse - userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{ + if err := userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{ UserID: device.UserID, Version: version, AuthData: kb.AuthData, Algorithm: kb.Algorithm, - }, &performKeyBackupResp) + }, &performKeyBackupResp); err != nil { + return jsonerror.InternalServerError() + } if performKeyBackupResp.Error != "" { if performKeyBackupResp.BadInput { return util.JSONResponse{ @@ -157,11 +161,13 @@ func ModifyKeyBackupVersionAuthData(req *http.Request, userAPI userapi.UserInter // Implements DELETE /_matrix/client/r0/room_keys/version/{version} func DeleteKeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version string) util.JSONResponse { var performKeyBackupResp userapi.PerformKeyBackupResponse - userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{ + if err := userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{ UserID: device.UserID, Version: version, DeleteBackup: true, - }, &performKeyBackupResp) + }, &performKeyBackupResp); err != nil { + return jsonerror.InternalServerError() + } if performKeyBackupResp.Error != "" { if performKeyBackupResp.BadInput { return util.JSONResponse{ @@ -191,11 +197,13 @@ func UploadBackupKeys( req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version string, keys *keyBackupSessionRequest, ) util.JSONResponse { var performKeyBackupResp userapi.PerformKeyBackupResponse - userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{ + if err := userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{ UserID: device.UserID, Version: version, Keys: *keys, - }, &performKeyBackupResp) + }, &performKeyBackupResp); err != nil && performKeyBackupResp.Error == "" { + return jsonerror.InternalServerError() + } if performKeyBackupResp.Error != "" { if performKeyBackupResp.BadInput { return util.JSONResponse{ diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index cdb106c23..975185796 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -158,10 +158,10 @@ func main() { asAPI := appservice.NewInternalAPI(&base.Base, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) fsAPI := federationapi.NewInternalAPI( - &base.Base, federation, rsAPI, userAPI, base.Base.Caches, true, + &base.Base, federation, rsAPI, userAPI, base.Base.Caches, nil, true, ) keyRing := fsAPI.KeyRing() - rsAPI.SetFederationAPI(fsAPI) + rsAPI.SetFederationAPI(fsAPI, keyRing) provider := newPublicRoomsProvider(base.LibP2PPubsub, rsAPI) err = provider.Start() if err != nil { diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index e912b8420..62833b2a0 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -185,7 +185,7 @@ func main() { rsComponent := roomserver.NewInternalAPI(base) rsAPI := rsComponent fsAPI := federationapi.NewInternalAPI( - base, federation, rsAPI, nil, base.Caches, true, + base, federation, rsAPI, nil, base.Caches, keyRing, true, ) keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI) @@ -199,8 +199,7 @@ func main() { asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) - rsComponent.SetFederationAPI(fsAPI) - rsComponent.SetKeyring(keyRing) + rsComponent.SetFederationAPI(fsAPI, keyRing) monolith := setup.Monolith{ Config: base.Cfg, diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 219254c42..1d5b77e7c 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -118,11 +118,10 @@ func main() { asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) fsAPI := federationapi.NewInternalAPI( - base, federation, rsAPI, userAPI, base.Caches, true, + base, federation, rsAPI, userAPI, base.Caches, keyRing, true, ) - rsComponent.SetFederationAPI(fsAPI) - rsComponent.SetKeyring(keyRing) + rsComponent.SetFederationAPI(fsAPI, keyRing) monolith := setup.Monolith{ Config: base.Cfg, diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index ab18179fd..65352a2b6 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -67,6 +67,7 @@ func main() { cfg.MediaAPI.InternalAPI.Connect = httpAPIAddr cfg.RoomServer.InternalAPI.Connect = httpAPIAddr cfg.SyncAPI.InternalAPI.Connect = httpAPIAddr + cfg.UserAPI.InternalAPI.Connect = httpAPIAddr options = append(options, basepkg.UseHTTPAPIs) } @@ -90,7 +91,7 @@ func main() { } fsAPI := federationapi.NewInternalAPI( - base, federation, rsAPI, nil, base.Caches, false, + base, federation, rsAPI, nil, base.Caches, nil, false, ) if base.UseHTTPAPIs { federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI) @@ -100,24 +101,44 @@ func main() { // The underlying roomserver implementation needs to be able to call the fedsender. // This is different to rsAPI which can be the http client which doesn't need this dependency - rsImpl.SetFederationAPI(fsAPI) + rsImpl.SetFederationAPI(fsAPI, keyRing) - keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI) - userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI) - keyAPI.SetUserAPI(userAPI) - fsAPI.SetUserAPI(userAPI) + keyImpl := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI) + keyAPI := keyImpl + if base.UseHTTPAPIs { + keyserver.AddInternalRoutes(base.InternalAPIMux, keyAPI) + keyAPI = base.KeyServerHTTPClient() + } + userImpl := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI) + userAPI := userImpl + if base.UseHTTPAPIs { + userapi.AddInternalRoutes(base.InternalAPIMux, userAPI) + userAPI = base.UserAPIClient() + } if traceInternal { userAPI = &uapi.UserInternalAPITrace{ Impl: userAPI, } } - // needs to be after the SetUserAPI call above + + // TODO: This should use userAPI, not userImpl, but the appservice setup races with + // the listeners and panics at startup if it tries to create appservice accounts + // before the listeners are up. + asAPI := appservice.NewInternalAPI(base, userImpl, rsAPI) if base.UseHTTPAPIs { - keyserver.AddInternalRoutes(base.InternalAPIMux, keyAPI) - keyAPI = base.KeyServerHTTPClient() + appservice.AddInternalRoutes(base.InternalAPIMux, asAPI) + asAPI = base.AppserviceHTTPClient() } + // The underlying roomserver implementation needs to be able to call the fedsender. + // This is different to rsAPI which can be the http client which doesn't need this + // dependency. Other components also need updating after their dependencies are up. + rsImpl.SetFederationAPI(fsAPI, keyRing) + rsImpl.SetAppserviceAPI(asAPI) + keyImpl.SetUserAPI(userAPI) + fsAPI.SetUserAPI(userAPI) + eduInputAPI := eduserver.NewInternalAPI( base, cache.New(), userAPI, ) @@ -126,13 +147,6 @@ func main() { eduInputAPI = base.EDUServerClient() } - asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) - if base.UseHTTPAPIs { - appservice.AddInternalRoutes(base.InternalAPIMux, asAPI) - asAPI = base.AppserviceHTTPClient() - } - rsAPI.SetAppserviceAPI(asAPI) - monolith := setup.Monolith{ Config: base.Cfg, AccountDB: accountDB, diff --git a/cmd/dendrite-polylith-multi/personalities/federationapi.go b/cmd/dendrite-polylith-multi/personalities/federationapi.go index 9b59cf45b..44357d660 100644 --- a/cmd/dendrite-polylith-multi/personalities/federationapi.go +++ b/cmd/dendrite-polylith-multi/personalities/federationapi.go @@ -23,10 +23,10 @@ import ( func FederationAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { userAPI := base.UserAPIClient() federation := base.CreateFederationClient() - fsAPI := base.FederationAPIHTTPClient() - keyRing := fsAPI.KeyRing() rsAPI := base.RoomserverHTTPClient() keyAPI := base.KeyServerHTTPClient() + fsAPI := federationapi.NewInternalAPI(base, federation, rsAPI, base.Caches, nil, true) + keyRing := fsAPI.KeyRing() federationapi.AddPublicRoutes( base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicWellKnownAPIMux, @@ -35,6 +35,8 @@ func FederationAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { &base.Cfg.MSCs, nil, ) + federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI) + base.SetupAndServeHTTP( base.Cfg.FederationAPI.InternalAPI.Listen, base.Cfg.FederationAPI.ExternalAPI.Listen, diff --git a/cmd/dendrite-polylith-multi/personalities/roomserver.go b/cmd/dendrite-polylith-multi/personalities/roomserver.go index 23514dbed..1deb51ce0 100644 --- a/cmd/dendrite-polylith-multi/personalities/roomserver.go +++ b/cmd/dendrite-polylith-multi/personalities/roomserver.go @@ -24,7 +24,7 @@ func RoomServer(base *basepkg.BaseDendrite, cfg *config.Dendrite) { asAPI := base.AppserviceHTTPClient() fsAPI := base.FederationAPIHTTPClient() rsAPI := roomserver.NewInternalAPI(base) - rsAPI.SetFederationAPI(fsAPI) + rsAPI.SetFederationAPI(fsAPI, fsAPI.KeyRing()) rsAPI.SetAppserviceAPI(asAPI) roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI) diff --git a/cmd/dendritejs-pinecone/main.go b/cmd/dendritejs-pinecone/main.go index 8665afbf6..776feef15 100644 --- a/cmd/dendritejs-pinecone/main.go +++ b/cmd/dendritejs-pinecone/main.go @@ -198,9 +198,9 @@ func startup() { base, userAPI, rsAPI, ) rsAPI.SetAppserviceAPI(asQuery) - fedSenderAPI := federationapi.NewInternalAPI(base, federation, rsAPI, userAPI, base.Caches, true) - rsAPI.SetFederationAPI(fedSenderAPI) - rsAPI.SetKeyring(keyRing) + + fedSenderAPI := federationapi.NewInternalAPI(base, federation, rsAPI, userAPI, base.Caches, keyRing, true) + rsAPI.SetFederationAPI(fedSenderAPI, keyRing) monolith := setup.Monolith{ Config: base.Cfg, diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 969581168..13a0872d0 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -210,9 +210,8 @@ func main() { base, userAPI, rsAPI, ) rsAPI.SetAppserviceAPI(asQuery) - fedSenderAPI := federationapi.NewInternalAPI(base, federation, rsAPI, base.Caches, true) - rsAPI.SetFederationAPI(fedSenderAPI) - rsAPI.SetKeyring(keyRing) + fedSenderAPI := federationapi.NewInternalAPI(base, federation, rsAPI, base.Caches, keyRing, true) + rsAPI.SetFederationAPI(fedSenderAPI, keyRing) p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node, fedSenderAPI, federation) monolith := setup.Monolith{ diff --git a/dendrite-config.yaml b/dendrite-config.yaml index 3e15f4b8e..2346674dd 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -378,9 +378,10 @@ tracing: baggage_restrictions: null throttler: null -# Logging configuration, in addition to the standard logging that is sent to -# stdout by Dendrite. +# Logging configuration logging: +- type: std + level: info - type: file level: info params: diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index a9c5f68d1..8f6ae4116 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -74,6 +74,7 @@ func NewInternalAPI( rsAPI roomserverAPI.RoomserverInternalAPI, userAPI userAPI.UserInternalAPI, caches *caching.Caches, + keyRing *gomatrixserverlib.KeyRing, resetBlacklist bool, ) api.FederationInternalAPI { cfg := &base.Cfg.FederationAPI @@ -126,5 +127,5 @@ func NewInternalAPI( logrus.WithError(err).Panic("failed to start key server consumer") } - return internal.NewFederationInternalAPI(federationDB, cfg, rsAPI, userAPI, federation, stats, caches, queues) + return internal.NewFederationInternalAPI(federationDB, cfg, rsAPI, userAPI, federation, stats, caches, queues, keyRing) } diff --git a/federationapi/federationapi_keys_test.go b/federationapi/federationapi_keys_test.go index f9af9c76b..667c0db91 100644 --- a/federationapi/federationapi_keys_test.go +++ b/federationapi/federationapi_keys_test.go @@ -94,7 +94,7 @@ func TestMain(m *testing.M) { // Finally, build the server key APIs. sbase := base.NewBaseDendrite(cfg, "Monolith", base.NoCacheMetrics) - s.api = NewInternalAPI(sbase, s.fedclient, nil, nil, s.cache, true) + s.api = NewInternalAPI(sbase, s.fedclient, nil, nil, s.cache, nil, true) } // Now that we have built our server key APIs, start the diff --git a/federationapi/internal/api.go b/federationapi/internal/api.go index 42ff2068a..940bcf6fa 100644 --- a/federationapi/internal/api.go +++ b/federationapi/internal/api.go @@ -42,58 +42,61 @@ func NewFederationInternalAPI( statistics *statistics.Statistics, caches *caching.Caches, queues *queue.OutgoingQueues, + keyRing *gomatrixserverlib.KeyRing, ) *FederationInternalAPI { serverKeyDB, err := cache.NewKeyDatabase(db, caches) if err != nil { logrus.WithError(err).Panicf("failed to set up caching wrapper for server key database") } - keyRing := &gomatrixserverlib.KeyRing{ - KeyFetchers: []gomatrixserverlib.KeyFetcher{}, - KeyDatabase: serverKeyDB, - } - - addDirectFetcher := func() { - keyRing.KeyFetchers = append( - keyRing.KeyFetchers, - &gomatrixserverlib.DirectKeyFetcher{ - Client: federation, - }, - ) - } - - if cfg.PreferDirectFetch { - addDirectFetcher() - } else { - defer addDirectFetcher() - } - - var b64e = base64.StdEncoding.WithPadding(base64.NoPadding) - for _, ps := range cfg.KeyPerspectives { - perspective := &gomatrixserverlib.PerspectiveKeyFetcher{ - PerspectiveServerName: ps.ServerName, - PerspectiveServerKeys: map[gomatrixserverlib.KeyID]ed25519.PublicKey{}, - Client: federation, + if keyRing == nil { + keyRing = &gomatrixserverlib.KeyRing{ + KeyFetchers: []gomatrixserverlib.KeyFetcher{}, + KeyDatabase: serverKeyDB, } - for _, key := range ps.Keys { - rawkey, err := b64e.DecodeString(key.PublicKey) - if err != nil { - logrus.WithError(err).WithFields(logrus.Fields{ - "server_name": ps.ServerName, - "public_key": key.PublicKey, - }).Warn("Couldn't parse perspective key") - continue + addDirectFetcher := func() { + keyRing.KeyFetchers = append( + keyRing.KeyFetchers, + &gomatrixserverlib.DirectKeyFetcher{ + Client: federation, + }, + ) + } + + if cfg.PreferDirectFetch { + addDirectFetcher() + } else { + defer addDirectFetcher() + } + + var b64e = base64.StdEncoding.WithPadding(base64.NoPadding) + for _, ps := range cfg.KeyPerspectives { + perspective := &gomatrixserverlib.PerspectiveKeyFetcher{ + PerspectiveServerName: ps.ServerName, + PerspectiveServerKeys: map[gomatrixserverlib.KeyID]ed25519.PublicKey{}, + Client: federation, } - perspective.PerspectiveServerKeys[key.KeyID] = rawkey + + for _, key := range ps.Keys { + rawkey, err := b64e.DecodeString(key.PublicKey) + if err != nil { + logrus.WithError(err).WithFields(logrus.Fields{ + "server_name": ps.ServerName, + "public_key": key.PublicKey, + }).Warn("Couldn't parse perspective key") + continue + } + perspective.PerspectiveServerKeys[key.KeyID] = rawkey + } + + keyRing.KeyFetchers = append(keyRing.KeyFetchers, perspective) + + logrus.WithFields(logrus.Fields{ + "server_name": ps.ServerName, + "num_public_keys": len(ps.Keys), + }).Info("Enabled perspective key fetcher") } - - keyRing.KeyFetchers = append(keyRing.KeyFetchers, perspective) - - logrus.WithFields(logrus.Fields{ - "server_name": ps.ServerName, - "num_public_keys": len(ps.Keys), - }).Info("Enabled perspective key fetcher") } return &FederationInternalAPI{ diff --git a/go.mod b/go.mod index 300d22673..5cd4b0e12 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ require ( github.com/Arceliar/ironwood v0.0.0-20210619124114-6ad55cae5031 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/HdrHistogram/hdrhistogram-go v1.0.1 // indirect + github.com/MFAshby/stdemuxerhook v1.0.0 // indirect github.com/Masterminds/semver/v3 v3.1.1 github.com/Shopify/sarama v1.29.1 github.com/codeclysm/extract v2.2.0+incompatible @@ -33,7 +34,7 @@ require ( github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 github.com/matrix-org/gomatrixserverlib v0.0.0-20211115192839-15a64d244aa2 github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 - github.com/matrix-org/pinecone v0.0.0-20211125101824-cc7886682cfd + github.com/matrix-org/pinecone v0.0.0-20211213132835-aa2808d77947 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.8 github.com/morikuni/aec v1.0.0 // indirect diff --git a/go.sum b/go.sum index d56241d35..f14e2e987 100644 --- a/go.sum +++ b/go.sum @@ -59,6 +59,8 @@ github.com/HdrHistogram/hdrhistogram-go v1.0.1 h1:GX8GAYDuhlFQnI2fRDHQhTlkHMz8bE github.com/HdrHistogram/hdrhistogram-go v1.0.1/go.mod h1:BWJ+nMSHY3L41Zj7CA3uXnloDp7xxV0YvstAE7nKTaM= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= +github.com/MFAshby/stdemuxerhook v1.0.0 h1:1XFGzakrsHMv76AeanPDL26NOgwjPl/OUxbGhJthwMc= +github.com/MFAshby/stdemuxerhook v1.0.0/go.mod h1:nLMI9FUf9Hz98n+yAXsTMUR4RZQy28uCTLG1Fzvj/uY= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= @@ -997,8 +999,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20211115192839-15a64d244aa2 h1:RF github.com/matrix-org/gomatrixserverlib v0.0.0-20211115192839-15a64d244aa2/go.mod h1:rB8tBUUUo1rzUqpzklRDSooxZ6YMhoaEPx4SO5fGeUc= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= -github.com/matrix-org/pinecone v0.0.0-20211125101824-cc7886682cfd h1:/iX6jehN2sO8n4pn63U+7iDoNx18fjC6pQ2RpwyZtKk= -github.com/matrix-org/pinecone v0.0.0-20211125101824-cc7886682cfd/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk= +github.com/matrix-org/pinecone v0.0.0-20211213132835-aa2808d77947 h1:TxO9TMFAuF+Vz3vZV53z5mjycWtF1+naY9ffs6QfZxc= +github.com/matrix-org/pinecone v0.0.0-20211213132835-aa2808d77947/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= diff --git a/internal/httputil/http.go b/internal/httputil/http.go index a469c8ac7..4527e2b95 100644 --- a/internal/httputil/http.go +++ b/internal/httputil/http.go @@ -23,6 +23,7 @@ import ( "net/url" "strings" + "github.com/matrix-org/dendrite/userapi/api" opentracing "github.com/opentracing/opentracing-go" "github.com/opentracing/opentracing-go/ext" ) @@ -72,6 +73,9 @@ func PostJSON( var errorBody struct { Message string `json:"message"` } + if _, ok := response.(*api.PerformKeyBackupResponse); ok { // TODO: remove this, once cross-boundary errors are a thing + return nil + } if msgerr := json.NewDecoder(res.Body).Decode(&errorBody); msgerr == nil { return fmt.Errorf("internal API: %d from %s: %s", res.StatusCode, apiURL, errorBody.Message) } diff --git a/internal/log_unix.go b/internal/log_unix.go index 25ad04205..1e1094f23 100644 --- a/internal/log_unix.go +++ b/internal/log_unix.go @@ -12,13 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !windows // +build !windows package internal import ( + "io/ioutil" "log/syslog" + "github.com/MFAshby/stdemuxerhook" "github.com/matrix-org/dendrite/setup/config" "github.com/sirupsen/logrus" lSyslog "github.com/sirupsen/logrus/hooks/syslog" @@ -28,7 +31,7 @@ import ( // If something fails here it means that the logging was improperly configured, // so we just exit with the error func SetupHookLogging(hooks []config.LogrusHook, componentName string) { - logrus.SetReportCaller(true) + stdLogAdded := false for _, hook := range hooks { // Check we received a proper logging level level, err := logrus.ParseLevel(hook.Level) @@ -49,10 +52,18 @@ func SetupHookLogging(hooks []config.LogrusHook, componentName string) { case "syslog": checkSyslogHookParams(hook.Params) setupSyslogHook(hook, level, componentName) + case "std": + setupStdLogHook(level) + stdLogAdded = true default: logrus.Fatalf("Unrecognised logging hook type: %s", hook.Type) } } + if !stdLogAdded { + setupStdLogHook(logrus.InfoLevel) + } + // Hooks are now configured for stdout/err, so throw away the default logger output + logrus.SetOutput(ioutil.Discard) } func checkSyslogHookParams(params map[string]interface{}) { @@ -76,6 +87,10 @@ func checkSyslogHookParams(params map[string]interface{}) { } +func setupStdLogHook(level logrus.Level) { + logrus.AddHook(&logLevelHook{level, stdemuxerhook.New(logrus.StandardLogger())}) +} + func setupSyslogHook(hook config.LogrusHook, level logrus.Level, componentName string) { syslogHook, err := lSyslog.NewSyslogHook(hook.Params["protocol"].(string), hook.Params["address"].(string), syslog.LOG_INFO, componentName) if err == nil { diff --git a/roomserver/api/api.go b/roomserver/api/api.go index 1e882ca10..d35fd84df 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -12,9 +12,8 @@ import ( type RoomserverInternalAPI interface { // needed to avoid chicken and egg scenario when setting up the // interdependencies between the roomserver and other input APIs - SetFederationAPI(fsAPI fsAPI.FederationInternalAPI) + SetFederationAPI(fsAPI fsAPI.FederationInternalAPI, keyRing *gomatrixserverlib.KeyRing) SetAppserviceAPI(asAPI asAPI.AppServiceQueryAPI) - SetKeyring(keyRing *gomatrixserverlib.KeyRing) InputRoomEvents( ctx context.Context, diff --git a/roomserver/api/api_trace.go b/roomserver/api/api_trace.go index cb8c471a3..64cbaca49 100644 --- a/roomserver/api/api_trace.go +++ b/roomserver/api/api_trace.go @@ -17,12 +17,8 @@ type RoomserverInternalAPITrace struct { Impl RoomserverInternalAPI } -func (t *RoomserverInternalAPITrace) SetKeyring(keyRing *gomatrixserverlib.KeyRing) { - t.Impl.SetKeyring(keyRing) -} - -func (t *RoomserverInternalAPITrace) SetFederationAPI(fsAPI fsAPI.FederationInternalAPI) { - t.Impl.SetFederationAPI(fsAPI) +func (t *RoomserverInternalAPITrace) SetFederationAPI(fsAPI fsAPI.FederationInternalAPI, keyRing *gomatrixserverlib.KeyRing) { + t.Impl.SetFederationAPI(fsAPI, keyRing) } func (t *RoomserverInternalAPITrace) SetAppserviceAPI(asAPI asAPI.AppServiceQueryAPI) { diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go index bda585cf8..67bbc7aba 100644 --- a/roomserver/internal/api.go +++ b/roomserver/internal/api.go @@ -73,18 +73,12 @@ func NewRoomserverAPI( return a } -// SetKeyring sets the keyring to a given keyring. This is only useful for the P2P -// demos and must be called after SetFederationSenderInputAPI. -func (r *RoomserverInternalAPI) SetKeyring(keyRing *gomatrixserverlib.KeyRing) { - r.KeyRing = keyRing -} - // SetFederationInputAPI passes in a federation input API reference so that we can // avoid the chicken-and-egg problem of both the roomserver input API and the // federation input API being interdependent. -func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.FederationInternalAPI) { +func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.FederationInternalAPI, keyRing *gomatrixserverlib.KeyRing) { r.fsAPI = fsAPI - r.SetKeyring(fsAPI.KeyRing()) + r.KeyRing = keyRing r.Inviter = &perform.Inviter{ DB: r.DB, diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index d8ce9727f..fc712f47b 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -122,7 +122,7 @@ func (r *Inputer) processRoomEvent( } // Store the event. - _, stateAtEvent, redactionEvent, redactedEventID, err := r.DB.StoreEvent(ctx, event, authEventNIDs, isRejected) + _, _, stateAtEvent, redactionEvent, redactedEventID, err := r.DB.StoreEvent(ctx, event, authEventNIDs, isRejected) if err != nil { return "", fmt.Errorf("r.DB.StoreEvent: %w", err) } diff --git a/roomserver/internal/perform/perform_backfill.go b/roomserver/internal/perform/perform_backfill.go index 8c2477dee..e198f67d8 100644 --- a/roomserver/internal/perform/perform_backfill.go +++ b/roomserver/internal/perform/perform_backfill.go @@ -546,6 +546,7 @@ func joinEventsFromHistoryVisibility( func persistEvents(ctx context.Context, db storage.Database, events []*gomatrixserverlib.HeaderedEvent) (types.RoomNID, map[string]types.Event) { var roomNID types.RoomNID + var eventNID types.EventNID backfilledEventMap := make(map[string]types.Event) for j, ev := range events { nidMap, err := db.EventNIDs(ctx, ev.AuthEventIDs()) @@ -559,10 +560,9 @@ func persistEvents(ctx context.Context, db storage.Database, events []*gomatrixs authNids[i] = nid i++ } - var stateAtEvent types.StateAtEvent var redactedEventID string var redactionEvent *gomatrixserverlib.Event - roomNID, stateAtEvent, redactionEvent, redactedEventID, err = db.StoreEvent(ctx, ev.Unwrap(), authNids, false) + eventNID, roomNID, _, redactionEvent, redactedEventID, err = db.StoreEvent(ctx, ev.Unwrap(), authNids, false) if err != nil { logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to persist event") continue @@ -581,7 +581,7 @@ func persistEvents(ctx context.Context, db storage.Database, events []*gomatrixs events[j] = ev } backfilledEventMap[ev.EventID()] = types.Event{ - EventNID: stateAtEvent.StateEntry.EventNID, + EventNID: eventNID, Event: ev.Unwrap(), } } diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go index 2afeb8b13..4f6a58bde 100644 --- a/roomserver/inthttp/client.go +++ b/roomserver/inthttp/client.go @@ -83,12 +83,8 @@ func NewRoomserverClient( }, nil } -// SetKeyring no-ops in HTTP client mode as there is no chicken/egg scenario -func (h *httpRoomserverInternalAPI) SetKeyring(keyRing *gomatrixserverlib.KeyRing) { -} - // SetFederationInputAPI no-ops in HTTP client mode as there is no chicken/egg scenario -func (h *httpRoomserverInternalAPI) SetFederationAPI(fsAPI fsInputAPI.FederationInternalAPI) { +func (h *httpRoomserverInternalAPI) SetFederationAPI(fsAPI fsInputAPI.FederationInternalAPI, keyRing *gomatrixserverlib.KeyRing) { } // SetAppserviceAPI no-ops in HTTP client mode as there is no chicken/egg scenario diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index 7f6b98557..15764366b 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -70,7 +70,7 @@ type Database interface { StoreEvent( ctx context.Context, event *gomatrixserverlib.Event, authEventNIDs []types.EventNID, isRejected bool, - ) (types.RoomNID, types.StateAtEvent, *gomatrixserverlib.Event, string, error) + ) (types.EventNID, types.RoomNID, types.StateAtEvent, *gomatrixserverlib.Event, string, error) // Look up the state entries for a list of string event IDs // Returns an error if the there is an error talking to the database // Returns a types.MissingEventError if the event IDs aren't in the database. diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index dbf706e5d..f49536f4e 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -461,7 +461,7 @@ func (d *Database) GetLatestEventsForUpdate( func (d *Database) StoreEvent( ctx context.Context, event *gomatrixserverlib.Event, authEventNIDs []types.EventNID, isRejected bool, -) (types.RoomNID, types.StateAtEvent, *gomatrixserverlib.Event, string, error) { +) (types.EventNID, types.RoomNID, types.StateAtEvent, *gomatrixserverlib.Event, string, error) { var ( roomNID types.RoomNID eventTypeNID types.EventTypeNID @@ -538,7 +538,7 @@ func (d *Database) StoreEvent( return nil }) if err != nil { - return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("d.Writer.Do: %w", err) + return 0, 0, types.StateAtEvent{}, nil, "", fmt.Errorf("d.Writer.Do: %w", err) } // We should attempt to update the previous events table with any @@ -551,10 +551,10 @@ func (d *Database) StoreEvent( if prevEvents := event.PrevEvents(); len(prevEvents) > 0 { roomInfo, err = d.RoomInfo(ctx, event.RoomID()) if err != nil { - return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("d.RoomInfo: %w", err) + return 0, 0, types.StateAtEvent{}, nil, "", fmt.Errorf("d.RoomInfo: %w", err) } if roomInfo == nil && len(prevEvents) > 0 { - return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("expected room %q to exist", event.RoomID()) + return 0, 0, types.StateAtEvent{}, nil, "", fmt.Errorf("expected room %q to exist", event.RoomID()) } // Create an updater - NB: on sqlite this WILL create a txn as we are directly calling the shared DB form of // GetLatestEventsForUpdate - not via the SQLiteDatabase form which has `nil` txns. This @@ -563,7 +563,7 @@ func (d *Database) StoreEvent( // to do writes however then this will need to go inside `Writer.Do`. updater, err = d.GetLatestEventsForUpdate(ctx, *roomInfo) if err != nil { - return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("NewLatestEventsUpdater: %w", err) + return 0, 0, types.StateAtEvent{}, nil, "", fmt.Errorf("NewLatestEventsUpdater: %w", err) } // Ensure that we atomically store prev events AND commit them. If we don't wrap StorePreviousEvents // and EndTransaction in a writer then it's possible for a new write txn to be made between the two @@ -580,11 +580,11 @@ func (d *Database) StoreEvent( return err }) if err != nil { - return 0, types.StateAtEvent{}, nil, "", err + return 0, 0, types.StateAtEvent{}, nil, "", err } } - return roomNID, types.StateAtEvent{ + return eventNID, roomNID, types.StateAtEvent{ BeforeStateSnapshotNID: stateNID, StateEntry: types.StateEntry{ StateKeyTuple: types.StateKeyTuple{ diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index b7fe7ee4f..3127eb17d 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -49,7 +49,8 @@ const eventsSchema = ` const insertEventSQL = ` INSERT INTO roomserver_events (room_nid, event_type_nid, event_state_key_nid, event_id, reference_sha256, auth_event_nids, depth, is_rejected) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) - ON CONFLICT DO NOTHING; + ON CONFLICT DO NOTHING + RETURNING event_nid, state_snapshot_nid; ` const selectEventSQL = "" + @@ -161,20 +162,13 @@ func (s *eventStatements) InsertEvent( ) (types.EventNID, types.StateSnapshotNID, error) { // attempt to insert: the last_row_id is the event NID var eventNID int64 + var stateNID int64 insertStmt := sqlutil.TxStmt(txn, s.insertEventStmt) - result, err := insertStmt.ExecContext( + err := insertStmt.QueryRowContext( ctx, int64(roomNID), int64(eventTypeNID), int64(eventStateKeyNID), eventID, referenceSHA256, eventNIDsAsArray(authEventNIDs), depth, isRejected, - ) - if err != nil { - return 0, 0, err - } - modified, err := result.RowsAffected() - if modified == 0 && err == nil { - return 0, 0, sql.ErrNoRows - } - eventNID, err = result.LastInsertId() - return types.EventNID(eventNID), 0, err + ).Scan(&eventNID, &stateNID) + return types.EventNID(eventNID), types.StateSnapshotNID(stateNID), err } func (s *eventStatements) SelectEvent( diff --git a/setup/base/base.go b/setup/base/base.go index 9ba88bef7..06c971170 100644 --- a/setup/base/base.go +++ b/setup/base/base.go @@ -21,7 +21,6 @@ import ( "io" "net" "net/http" - "net/url" "os" "os/signal" "syscall" @@ -79,7 +78,6 @@ type BaseDendrite struct { SynapseAdminMux *mux.Router UseHTTPAPIs bool apiHttpClient *http.Client - httpClient *http.Client Cfg *config.Dendrite Caches *caching.Caches DNSCache *gomatrixserverlib.DNSCache @@ -183,13 +181,6 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base }, }, } - client := http.Client{Timeout: HTTPClientTimeout} - if cfg.FederationAPI.Proxy.Enabled { - client.Transport = &http.Transport{Proxy: http.ProxyURL(&url.URL{ - Scheme: cfg.FederationAPI.Proxy.Protocol, - Host: fmt.Sprintf("%s:%d", cfg.FederationAPI.Proxy.Host, cfg.FederationAPI.Proxy.Port), - })} - } // Ideally we would only use SkipClean on routes which we know can allow '/' but due to // https://github.com/gorilla/mux/issues/460 we have to attach this at the top router. @@ -219,7 +210,6 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base InternalAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.InternalPathPrefix).Subrouter().UseEncodedPath(), SynapseAdminMux: mux.NewRouter().SkipClean(true).PathPrefix("/_synapse/").Subrouter().UseEncodedPath(), apiHttpClient: &apiClient, - httpClient: &client, } } diff --git a/setup/mscs/msc2836/msc2836_test.go b/setup/mscs/msc2836/msc2836_test.go index 18ab08be0..9044823af 100644 --- a/setup/mscs/msc2836/msc2836_test.go +++ b/setup/mscs/msc2836/msc2836_test.go @@ -7,7 +7,6 @@ import ( "crypto/sha256" "encoding/base64" "encoding/json" - "fmt" "io/ioutil" "net/http" "sort" @@ -504,7 +503,7 @@ type testUserAPI struct { func (u *testUserAPI) QueryAccessToken(ctx context.Context, req *userapi.QueryAccessTokenRequest, res *userapi.QueryAccessTokenResponse) error { dev, ok := u.accessTokens[req.AccessToken] if !ok { - res.Err = fmt.Errorf("unknown token") + res.Err = "unknown token" return nil } res.Device = &dev diff --git a/setup/mscs/msc2946/msc2946_test.go b/setup/mscs/msc2946/msc2946_test.go index 441892f3e..e8066c34d 100644 --- a/setup/mscs/msc2946/msc2946_test.go +++ b/setup/mscs/msc2946/msc2946_test.go @@ -19,7 +19,6 @@ import ( "context" "crypto/ed25519" "encoding/json" - "fmt" "io/ioutil" "net/http" "net/url" @@ -347,7 +346,7 @@ type testUserAPI struct { func (u *testUserAPI) QueryAccessToken(ctx context.Context, req *userapi.QueryAccessTokenRequest, res *userapi.QueryAccessTokenResponse) error { dev, ok := u.accessTokens[req.AccessToken] if !ok { - res.Err = fmt.Errorf("unknown token") + res.Err = "unknown token" return nil } res.Device = &dev diff --git a/syncapi/consumers/eduserver_typing.go b/syncapi/consumers/eduserver_typing.go index c10d1e94e..8d06e3ca8 100644 --- a/syncapi/consumers/eduserver_typing.go +++ b/syncapi/consumers/eduserver_typing.go @@ -74,6 +74,7 @@ func NewOutputTypingEventConsumer( func (s *OutputTypingEventConsumer) Start() error { s.eduCache.SetTimeoutCallback(func(userID, roomID string, latestSyncPosition int64) { pos := types.StreamPosition(latestSyncPosition) + s.stream.Advance(pos) s.notifier.OnNewTyping(roomID, types.StreamingToken{TypingPosition: pos}) }) return s.typingConsumer.Start() diff --git a/userapi/api/api.go b/userapi/api/api.go index d180d16f5..15c500ff4 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -35,7 +35,7 @@ type UserInternalAPI interface { PerformDeviceUpdate(ctx context.Context, req *PerformDeviceUpdateRequest, res *PerformDeviceUpdateResponse) error PerformAccountDeactivation(ctx context.Context, req *PerformAccountDeactivationRequest, res *PerformAccountDeactivationResponse) error PerformOpenIDTokenCreation(ctx context.Context, req *PerformOpenIDTokenCreationRequest, res *PerformOpenIDTokenCreationResponse) error - PerformKeyBackup(ctx context.Context, req *PerformKeyBackupRequest, res *PerformKeyBackupResponse) + PerformKeyBackup(ctx context.Context, req *PerformKeyBackupRequest, res *PerformKeyBackupResponse) error QueryKeyBackup(ctx context.Context, req *QueryKeyBackupRequest, res *QueryKeyBackupResponse) QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error QueryAccessToken(ctx context.Context, req *QueryAccessTokenRequest, res *QueryAccessTokenResponse) error @@ -186,7 +186,7 @@ type QueryAccessTokenRequest struct { // QueryAccessTokenResponse is the response for QueryAccessToken type QueryAccessTokenResponse struct { Device *Device - Err error // e.g ErrorForbidden + Err string // e.g ErrorForbidden } // QueryAccountDataRequest is the request for QueryAccountData @@ -295,6 +295,10 @@ type PerformDeviceCreationRequest struct { IPAddr string // Useragent for this device UserAgent string + // NoDeviceListUpdate determines whether we should avoid sending a device list + // update for this account. Generally the only reason to do this is if the account + // is an appservice account. + NoDeviceListUpdate bool } // PerformDeviceCreationResponse is the response for PerformDeviceCreation diff --git a/userapi/api/api_trace.go b/userapi/api/api_trace.go index 3ab1a7005..0a02313ee 100644 --- a/userapi/api/api_trace.go +++ b/userapi/api/api_trace.go @@ -74,11 +74,14 @@ func (t *UserInternalAPITrace) PerformOpenIDTokenCreation(ctx context.Context, r util.GetLogger(ctx).Infof("PerformOpenIDTokenCreation req=%+v res=%+v", js(req), js(res)) return err } -func (t *UserInternalAPITrace) PerformKeyBackup(ctx context.Context, req *PerformKeyBackupRequest, res *PerformKeyBackupResponse) { - t.Impl.PerformKeyBackup(ctx, req, res) +func (t *UserInternalAPITrace) PerformKeyBackup(ctx context.Context, req *PerformKeyBackupRequest, res *PerformKeyBackupResponse) error { + err := t.Impl.PerformKeyBackup(ctx, req, res) + util.GetLogger(ctx).Infof("PerformKeyBackup req=%+v res=%+v", js(req), js(res)) + return err } func (t *UserInternalAPITrace) QueryKeyBackup(ctx context.Context, req *QueryKeyBackupRequest, res *QueryKeyBackupResponse) { t.Impl.QueryKeyBackup(ctx, req, res) + util.GetLogger(ctx).Infof("QueryKeyBackup req=%+v res=%+v", js(req), js(res)) } func (t *UserInternalAPITrace) QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error { err := t.Impl.QueryProfile(ctx, req, res) diff --git a/userapi/internal/api.go b/userapi/internal/api.go index 7a94d8898..58a7dfbf1 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -130,6 +130,9 @@ func (a *UserInternalAPI) PerformDeviceCreation(ctx context.Context, req *api.Pe } res.DeviceCreated = true res.Device = dev + if req.NoDeviceListUpdate { + return nil + } // create empty device keys and upload them to trigger device list changes return a.deviceListUpdate(dev.UserID, []string{dev.ID}) } @@ -369,8 +372,11 @@ func (a *UserInternalAPI) QueryAccountData(ctx context.Context, req *api.QueryAc func (a *UserInternalAPI) QueryAccessToken(ctx context.Context, req *api.QueryAccessTokenRequest, res *api.QueryAccessTokenResponse) error { if req.AppServiceUserID != "" { appServiceDevice, err := a.queryAppServiceToken(ctx, req.AccessToken, req.AppServiceUserID) + if err != nil { + res.Err = err.Error() + } res.Device = appServiceDevice - res.Err = err + return nil } device, err := a.DeviceDB.GetDeviceByAccessToken(ctx, req.AccessToken) @@ -525,13 +531,16 @@ func (a *UserInternalAPI) QueryPresenceAfter(ctx context.Context, req *api.Query return nil } -func (a *UserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.PerformKeyBackupRequest, res *api.PerformKeyBackupResponse) { +func (a *UserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.PerformKeyBackupRequest, res *api.PerformKeyBackupResponse) error { // Delete metadata if req.DeleteBackup { if req.Version == "" { res.BadInput = true res.Error = "must specify a version to delete" - return + if res.Error != "" { + return fmt.Errorf(res.Error) + } + return nil } exists, err := a.AccountDB.DeleteKeyBackup(ctx, req.UserID, req.Version) if err != nil { @@ -539,7 +548,10 @@ func (a *UserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.Perform } res.Exists = exists res.Version = req.Version - return + if res.Error != "" { + return fmt.Errorf(res.Error) + } + return nil } // Create metadata if req.Version == "" { @@ -549,7 +561,10 @@ func (a *UserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.Perform } res.Exists = err == nil res.Version = version - return + if res.Error != "" { + return fmt.Errorf(res.Error) + } + return nil } // Update metadata if len(req.Keys.Rooms) == 0 { @@ -559,10 +574,17 @@ func (a *UserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.Perform } res.Exists = err == nil res.Version = req.Version - return + if res.Error != "" { + return fmt.Errorf(res.Error) + } + return nil } // Upload Keys for a specific version metadata a.uploadBackupKeys(ctx, req, res) + if res.Error != "" { + return fmt.Errorf(res.Error) + } + return nil } func (a *UserInternalAPI) uploadBackupKeys(ctx context.Context, req *api.PerformKeyBackupRequest, res *api.PerformKeyBackupResponse) { diff --git a/userapi/inthttp/client.go b/userapi/inthttp/client.go index 124374b63..29ba43d9f 100644 --- a/userapi/inthttp/client.go +++ b/userapi/inthttp/client.go @@ -248,7 +248,7 @@ func (h *httpUserInternalAPI) QueryPresenceForUser(ctx context.Context, req *api return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res) } -func (h *httpUserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.PerformKeyBackupRequest, res *api.PerformKeyBackupResponse) { +func (h *httpUserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.PerformKeyBackupRequest, res *api.PerformKeyBackupResponse) error { span, ctx := opentracing.StartSpanFromContext(ctx, "PerformKeyBackup") defer span.Finish() @@ -257,6 +257,7 @@ func (h *httpUserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.Per if err != nil { res.Error = err.Error() } + return nil } func (h *httpUserInternalAPI) QueryKeyBackup(ctx context.Context, req *api.QueryKeyBackupRequest, res *api.QueryKeyBackupResponse) { diff --git a/userapi/inthttp/server.go b/userapi/inthttp/server.go index c3cc17b16..ff1a473d8 100644 --- a/userapi/inthttp/server.go +++ b/userapi/inthttp/server.go @@ -16,6 +16,7 @@ package inthttp import ( "encoding/json" + "fmt" "net/http" "github.com/gorilla/mux" @@ -234,6 +235,34 @@ func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + internalAPIMux.Handle(QueryKeyBackupPath, + httputil.MakeInternalAPI("queryKeyBackup", func(req *http.Request) util.JSONResponse { + request := api.QueryKeyBackupRequest{} + response := api.QueryKeyBackupResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + s.QueryKeyBackup(req.Context(), &request, &response) + if response.Error != "" { + return util.ErrorResponse(fmt.Errorf("QueryKeyBackup: %s", response.Error)) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle(PerformKeyBackupPath, + httputil.MakeInternalAPI("performKeyBackup", func(req *http.Request) util.JSONResponse { + request := api.PerformKeyBackupRequest{} + response := api.PerformKeyBackupResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + err := s.PerformKeyBackup(req.Context(), &request, &response) + if err != nil { + return util.JSONResponse{Code: http.StatusBadRequest, JSON: &response} + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) internalAPIMux.Handle(InputPresenceDataPath, httputil.MakeInternalAPI("inputPresenceDataPath", func(req *http.Request) util.JSONResponse { request := api.InputPresenceRequest{}