From 6a1111c3d44cfa6b5301fed70045b1e555d39c65 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 6 Mar 2020 16:58:10 +0000 Subject: [PATCH 1/4] Try to recursively find auth events (to a point) if they are missing (#881) * Try to recursively find auth events (to a point) if they are missing * Remove recursion limit for now and other review fixes * Simplify error handling for recursion * Pass room version 1 only to MakeJoin until room version support comes later --- clientapi/routing/joinroom.go | 2 +- federationapi/routing/send.go | 17 +++++++++++++++++ go.mod | 4 ++-- go.sum | 18 ++++++------------ 4 files changed, 26 insertions(+), 15 deletions(-) diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index de9667e2e..7643c41f3 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -299,7 +299,7 @@ func (r joinRoomReq) joinRoomUsingServers( // server was invalid this returns an error. // Otherwise this returns a JSONResponse. func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib.ServerName) (*util.JSONResponse, error) { - respMakeJoin, err := r.federation.MakeJoin(r.req.Context(), server, roomID, r.userID) + respMakeJoin, err := r.federation.MakeJoin(r.req.Context(), server, roomID, r.userID, []int{1}) if err != nil { // TODO: Check if the user was not allowed to join the room. return nil, err diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 191e13ef7..079e121f9 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -211,7 +211,24 @@ func (t *txnReq) processEventWithMissingState(e gomatrixserverlib.Event) error { return err } // Check that the event is allowed by the state. +retryAllowedState: if err := checkAllowedByState(e, state.StateEvents); err != nil { + switch missing := err.(type) { + case gomatrixserverlib.MissingAuthEventError: + // An auth event was missing so let's look up that event over federation + for _, s := range state.StateEvents { + if s.EventID() != missing.AuthEventID { + continue + } + err = t.processEventWithMissingState(s) + // If there was no error retrieving the event from federation then + // we assume that it succeeded, so retry the original state check + if err == nil { + goto retryAllowedState + } + } + default: + } return err } // pass the event along with the state to the roomserver diff --git a/go.mod b/go.mod index 7d59aeb1b..924c4b67d 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200304160008-4ec1129a00c4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200304110715-894c3c86ce3e + github.com/matrix-org/gomatrixserverlib v0.0.0-20200306154041-df6bb9a3e424 github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible @@ -17,7 +17,7 @@ require ( github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v1.4.1 github.com/sirupsen/logrus v1.4.2 - github.com/tidwall/gjson v1.6.0 + github.com/tidwall/gjson v1.6.0 // indirect github.com/tidwall/pretty v1.0.1 // indirect github.com/uber/jaeger-client-go v2.22.1+incompatible github.com/uber/jaeger-lib v2.2.0+incompatible diff --git a/go.sum b/go.sum index 19dc9662c..3a509debb 100644 --- a/go.sum +++ b/go.sum @@ -44,6 +44,7 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= @@ -70,14 +71,8 @@ github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/matrix-org/dendrite v0.0.0-20200220135450-0352f250b857/go.mod h1:DZ35IoR+ViBNVPe9umdlOSnjvKl7wfyRmZg4QfWGvTo= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= -github.com/matrix-org/go-http-js-libp2p v0.0.0-20200225225149-e7191ca90a94 h1:gUqPXKWbuwmPnx3PJBxtQw5Ff8V229wZ7Ee0GaH/bg4= -github.com/matrix-org/go-http-js-libp2p v0.0.0-20200225225149-e7191ca90a94/go.mod h1:PJof7UbOKmVEEMsGqvbyIK0ldMMBjPH5EYia7MHR2RQ= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200304160008-4ec1129a00c4 h1:oFjG7X1jS473zPDix1/FBZ2qd0anM1Ko4AlJCB6MUZs= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200304160008-4ec1129a00c4/go.mod h1:PJof7UbOKmVEEMsGqvbyIK0ldMMBjPH5EYia7MHR2RQ= -github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074 h1:UWz6vfhmQVshBuE67X1BCsdMhEDtd+uOz8CJ48Fc0F4= -github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= -github.com/matrix-org/go-sqlite3-js v0.0.0-20200304163011-cfb4884075db h1:ERuFJq4DI8fakfBZlvXHltHZ0ix3K5YsLG0tQfQn6TI= -github.com/matrix-org/go-sqlite3-js v0.0.0-20200304163011-cfb4884075db/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 h1:UlhTKClOgWnSB25Rv+BS/Vc1mRinjNUErfyGEVOBP04= github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af h1:piaIBNQGIHnni27xRB7VKkEwoWCgAmeuYf8pxAyG0bI= @@ -86,14 +81,10 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200228131708-347eb77397b8 h1:a9IV2iKMznKJ16MmcG/NU7qMcZ4jIKmPXC6RkOZZq+Q= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200228131708-347eb77397b8/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200304110715-894c3c86ce3e h1:DA1lP2mB2ddd2PhMOaNPwRJFi/3aL2Lj7bxnAhFxBFQ= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200304110715-894c3c86ce3e/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200306154041-df6bb9a3e424 h1:H61lT6ckUFIDl9qb636qNomfo0B52lFt73ecioiqF10= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200306154041-df6bb9a3e424/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= -github.com/matrix-org/pq v1.3.2 h1:7Kh2Qz4xonRH8OcFpCIj2najpBxF2+j1ff0hC082L68= -github.com/matrix-org/pq v1.3.2/go.mod h1:l6tPTzDjcj8fhD5OD0+7dejtZrwyjJadAAYM+CntUVQ= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5/go.mod h1:lePuOiXLNDott7NZfnQvJk0lAZ5HgvIuWGhel6J+RLA= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= @@ -200,6 +191,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -212,6 +204,7 @@ golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/Shopify/sarama.v1 v1.20.1 h1:Gi09A3fJXm0Jgt8kuKZ8YK+r60GfYn7MQuEmI3oq6hE= gopkg.in/Shopify/sarama.v1 v1.20.1/go.mod h1:AxnvoaevB2nBjNK17cG61A3LleFcWFwVBHBt+cot4Oc= @@ -220,6 +213,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/h2non/bimg.v1 v1.0.18 h1:qn6/RpBHt+7WQqoBcK+aF2puc6nC78eZj5LexxoalT4= gopkg.in/h2non/bimg.v1 v1.0.18/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= From c31cb0227111ab4441d6282081ebd6494a422ba8 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 6 Mar 2020 18:00:07 +0000 Subject: [PATCH 2/4] bugfix: Fix a race condition when creating guest accounts (#882) * bugfix: Fix a race condition when creating guest accounts It was possible to both select the same next numeric ID and then both attempt to INSERT this into the table. This would cause a UNIQUE violation which then presented itself as an error in sqlite because it does not implement `common.IsUniqueConstraintViolationErr`. The fix here is NOT to implement `common.IsUniqueConstraintViolationErr` otherwise the 2 users would get the SAME guest account. Instead, all of these operations should be done inside a transaction. This is what this PR does. * Update postgres * Typo * Actually use the txn when creating accounts * bugfix for database is locked on guest reg --- clientapi/auth/storage/accounts/interface.go | 1 + .../accounts/postgres/account_data_table.go | 4 +- .../accounts/postgres/accounts_table.go | 12 +++-- .../accounts/postgres/profile_table.go | 4 +- .../auth/storage/accounts/postgres/storage.go | 40 ++++++++++++-- .../accounts/sqlite3/account_data_table.go | 5 +- .../accounts/sqlite3/accounts_table.go | 14 +++-- .../storage/accounts/sqlite3/profile_table.go | 4 +- .../auth/storage/accounts/sqlite3/storage.go | 53 ++++++++++++++++--- clientapi/routing/register.go | 11 +--- 10 files changed, 108 insertions(+), 40 deletions(-) diff --git a/clientapi/auth/storage/accounts/interface.go b/clientapi/auth/storage/accounts/interface.go index 83d3ee725..9f6e3e1ea 100644 --- a/clientapi/auth/storage/accounts/interface.go +++ b/clientapi/auth/storage/accounts/interface.go @@ -30,6 +30,7 @@ type Database interface { SetAvatarURL(ctx context.Context, localpart string, avatarURL string) error SetDisplayName(ctx context.Context, localpart string, displayName string) error CreateAccount(ctx context.Context, localpart, plaintextPassword, appserviceID string) (*authtypes.Account, error) + CreateGuestAccount(ctx context.Context) (*authtypes.Account, error) UpdateMemberships(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string) error GetMembershipInRoomByLocalpart(ctx context.Context, localpart, roomID string) (authtypes.Membership, error) GetMembershipsByLocalpart(ctx context.Context, localpart string) (memberships []authtypes.Membership, err error) diff --git a/clientapi/auth/storage/accounts/postgres/account_data_table.go b/clientapi/auth/storage/accounts/postgres/account_data_table.go index d0cfcc0cf..4573999b4 100644 --- a/clientapi/auth/storage/accounts/postgres/account_data_table.go +++ b/clientapi/auth/storage/accounts/postgres/account_data_table.go @@ -72,9 +72,9 @@ func (s *accountDataStatements) prepare(db *sql.DB) (err error) { } func (s *accountDataStatements) insertAccountData( - ctx context.Context, localpart, roomID, dataType, content string, + ctx context.Context, txn *sql.Tx, localpart, roomID, dataType, content string, ) (err error) { - stmt := s.insertAccountDataStmt + stmt := txn.Stmt(s.insertAccountDataStmt) _, err = stmt.ExecContext(ctx, localpart, roomID, dataType, content) return } diff --git a/clientapi/auth/storage/accounts/postgres/accounts_table.go b/clientapi/auth/storage/accounts/postgres/accounts_table.go index 6b8ed3728..85c1938a1 100644 --- a/clientapi/auth/storage/accounts/postgres/accounts_table.go +++ b/clientapi/auth/storage/accounts/postgres/accounts_table.go @@ -91,10 +91,10 @@ func (s *accountsStatements) prepare(db *sql.DB, server gomatrixserverlib.Server // this account will be passwordless. Returns an error if this account already exists. Returns the account // on success. func (s *accountsStatements) insertAccount( - ctx context.Context, localpart, hash, appserviceID string, + ctx context.Context, txn *sql.Tx, localpart, hash, appserviceID string, ) (*authtypes.Account, error) { createdTimeMS := time.Now().UnixNano() / 1000000 - stmt := s.insertAccountStmt + stmt := txn.Stmt(s.insertAccountStmt) var err error if appserviceID == "" { @@ -146,8 +146,12 @@ func (s *accountsStatements) selectAccountByLocalpart( } func (s *accountsStatements) selectNewNumericLocalpart( - ctx context.Context, + ctx context.Context, txn *sql.Tx, ) (id int64, err error) { - err = s.selectNewNumericLocalpartStmt.QueryRowContext(ctx).Scan(&id) + stmt := s.selectNewNumericLocalpartStmt + if txn != nil { + stmt = txn.Stmt(stmt) + } + err = stmt.QueryRowContext(ctx).Scan(&id) return } diff --git a/clientapi/auth/storage/accounts/postgres/profile_table.go b/clientapi/auth/storage/accounts/postgres/profile_table.go index 38c76c40f..d2cbeb8e6 100644 --- a/clientapi/auth/storage/accounts/postgres/profile_table.go +++ b/clientapi/auth/storage/accounts/postgres/profile_table.go @@ -73,9 +73,9 @@ func (s *profilesStatements) prepare(db *sql.DB) (err error) { } func (s *profilesStatements) insertProfile( - ctx context.Context, localpart string, + ctx context.Context, txn *sql.Tx, localpart string, ) (err error) { - _, err = s.insertProfileStmt.ExecContext(ctx, localpart, "", "") + _, err = txn.Stmt(s.insertProfileStmt).ExecContext(ctx, localpart, "", "") return } diff --git a/clientapi/auth/storage/accounts/postgres/storage.go b/clientapi/auth/storage/accounts/postgres/storage.go index cb74d1315..8115dca43 100644 --- a/clientapi/auth/storage/accounts/postgres/storage.go +++ b/clientapi/auth/storage/accounts/postgres/storage.go @@ -18,6 +18,7 @@ import ( "context" "database/sql" "errors" + "strconv" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/common" @@ -118,11 +119,37 @@ func (d *Database) SetDisplayName( return d.profiles.setDisplayName(ctx, localpart, displayName) } +// CreateGuestAccount makes a new guest account and creates an empty profile +// for this account. +func (d *Database) CreateGuestAccount(ctx context.Context) (acc *authtypes.Account, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + var numLocalpart int64 + numLocalpart, err = d.accounts.selectNewNumericLocalpart(ctx, txn) + if err != nil { + return err + } + localpart := strconv.FormatInt(numLocalpart, 10) + acc, err = d.createAccount(ctx, txn, localpart, "", "") + return err + }) + return acc, err +} + // CreateAccount makes a new account with the given login name and password, and creates an empty profile // for this account. If no password is supplied, the account will be a passwordless account. If the // account already exists, it will return nil, nil. func (d *Database) CreateAccount( ctx context.Context, localpart, plaintextPassword, appserviceID string, +) (acc *authtypes.Account, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + acc, err = d.createAccount(ctx, txn, localpart, plaintextPassword, appserviceID) + return err + }) + return +} + +func (d *Database) createAccount( + ctx context.Context, txn *sql.Tx, localpart, plaintextPassword, appserviceID string, ) (*authtypes.Account, error) { var err error @@ -134,13 +161,14 @@ func (d *Database) CreateAccount( return nil, err } } - if err := d.profiles.insertProfile(ctx, localpart); err != nil { + if err := d.profiles.insertProfile(ctx, txn, localpart); err != nil { if common.IsUniqueConstraintViolationErr(err) { return nil, nil } return nil, err } - if err := d.SaveAccountData(ctx, localpart, "", "m.push_rules", `{ + + if err := d.accountDatas.insertAccountData(ctx, txn, localpart, "", "m.push_rules", `{ "global": { "content": [], "override": [], @@ -151,7 +179,7 @@ func (d *Database) CreateAccount( }`); err != nil { return nil, err } - return d.accounts.insertAccount(ctx, localpart, hash, appserviceID) + return d.accounts.insertAccount(ctx, txn, localpart, hash, appserviceID) } // SaveMembership saves the user matching a given localpart as a member of a given @@ -258,7 +286,9 @@ func (d *Database) newMembership( func (d *Database) SaveAccountData( ctx context.Context, localpart, roomID, dataType, content string, ) error { - return d.accountDatas.insertAccountData(ctx, localpart, roomID, dataType, content) + return common.WithTransaction(d.db, func(txn *sql.Tx) error { + return d.accountDatas.insertAccountData(ctx, txn, localpart, roomID, dataType, content) + }) } // GetAccountData returns account data related to a given localpart @@ -288,7 +318,7 @@ func (d *Database) GetAccountDataByType( func (d *Database) GetNewNumericLocalpart( ctx context.Context, ) (int64, error) { - return d.accounts.selectNewNumericLocalpart(ctx) + return d.accounts.selectNewNumericLocalpart(ctx, nil) } func hashPassword(plaintext string) (hash string, err error) { diff --git a/clientapi/auth/storage/accounts/sqlite3/account_data_table.go b/clientapi/auth/storage/accounts/sqlite3/account_data_table.go index c2143881b..b6bb63617 100644 --- a/clientapi/auth/storage/accounts/sqlite3/account_data_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/account_data_table.go @@ -72,10 +72,9 @@ func (s *accountDataStatements) prepare(db *sql.DB) (err error) { } func (s *accountDataStatements) insertAccountData( - ctx context.Context, localpart, roomID, dataType, content string, + ctx context.Context, txn *sql.Tx, localpart, roomID, dataType, content string, ) (err error) { - stmt := s.insertAccountDataStmt - _, err = stmt.ExecContext(ctx, localpart, roomID, dataType, content) + _, err = txn.Stmt(s.insertAccountDataStmt).ExecContext(ctx, localpart, roomID, dataType, content) return } diff --git a/clientapi/auth/storage/accounts/sqlite3/accounts_table.go b/clientapi/auth/storage/accounts/sqlite3/accounts_table.go index b029951f1..fd6a09cde 100644 --- a/clientapi/auth/storage/accounts/sqlite3/accounts_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/accounts_table.go @@ -89,16 +89,16 @@ func (s *accountsStatements) prepare(db *sql.DB, server gomatrixserverlib.Server // this account will be passwordless. Returns an error if this account already exists. Returns the account // on success. func (s *accountsStatements) insertAccount( - ctx context.Context, localpart, hash, appserviceID string, + ctx context.Context, txn *sql.Tx, localpart, hash, appserviceID string, ) (*authtypes.Account, error) { createdTimeMS := time.Now().UnixNano() / 1000000 stmt := s.insertAccountStmt var err error if appserviceID == "" { - _, err = stmt.ExecContext(ctx, localpart, createdTimeMS, hash, nil) + _, err = txn.Stmt(stmt).ExecContext(ctx, localpart, createdTimeMS, hash, nil) } else { - _, err = stmt.ExecContext(ctx, localpart, createdTimeMS, hash, appserviceID) + _, err = txn.Stmt(stmt).ExecContext(ctx, localpart, createdTimeMS, hash, appserviceID) } if err != nil { return nil, err @@ -144,8 +144,12 @@ func (s *accountsStatements) selectAccountByLocalpart( } func (s *accountsStatements) selectNewNumericLocalpart( - ctx context.Context, + ctx context.Context, txn *sql.Tx, ) (id int64, err error) { - err = s.selectNewNumericLocalpartStmt.QueryRowContext(ctx).Scan(&id) + stmt := s.selectNewNumericLocalpartStmt + if txn != nil { + stmt = txn.Stmt(stmt) + } + err = stmt.QueryRowContext(ctx).Scan(&id) return } diff --git a/clientapi/auth/storage/accounts/sqlite3/profile_table.go b/clientapi/auth/storage/accounts/sqlite3/profile_table.go index 7af8307e1..9b5192a02 100644 --- a/clientapi/auth/storage/accounts/sqlite3/profile_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/profile_table.go @@ -73,9 +73,9 @@ func (s *profilesStatements) prepare(db *sql.DB) (err error) { } func (s *profilesStatements) insertProfile( - ctx context.Context, localpart string, + ctx context.Context, txn *sql.Tx, localpart string, ) (err error) { - _, err = s.insertProfileStmt.ExecContext(ctx, localpart, "", "") + _, err = txn.Stmt(s.insertProfileStmt).ExecContext(ctx, localpart, "", "") return } diff --git a/clientapi/auth/storage/accounts/sqlite3/storage.go b/clientapi/auth/storage/accounts/sqlite3/storage.go index 3e62d10dd..9124640c6 100644 --- a/clientapi/auth/storage/accounts/sqlite3/storage.go +++ b/clientapi/auth/storage/accounts/sqlite3/storage.go @@ -18,6 +18,8 @@ import ( "context" "database/sql" "errors" + "strconv" + "sync" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/common" @@ -39,6 +41,8 @@ type Database struct { threepids threepidStatements filter filterStatements serverName gomatrixserverlib.ServerName + + createGuestAccountMu sync.Mutex } // NewDatabase creates a new accounts and profiles database @@ -76,7 +80,7 @@ func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) if err = f.prepare(db); err != nil { return nil, err } - return &Database{db, partitions, a, p, m, ac, t, f, serverName}, nil + return &Database{db, partitions, a, p, m, ac, t, f, serverName, sync.Mutex{}}, nil } // GetAccountByPassword returns the account associated with the given localpart and password. @@ -118,14 +122,46 @@ func (d *Database) SetDisplayName( return d.profiles.setDisplayName(ctx, localpart, displayName) } +// CreateGuestAccount makes a new guest account and creates an empty profile +// for this account. +func (d *Database) CreateGuestAccount(ctx context.Context) (acc *authtypes.Account, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + // We need to lock so we sequentially create numeric localparts. If we don't, two calls to + // this function will cause the same number to be selected and one will fail with 'database is locked' + // when the first txn upgrades to a write txn. + // We know we'll be the only process since this is sqlite ;) so a lock here will be all that is needed. + d.createGuestAccountMu.Lock() + defer d.createGuestAccountMu.Unlock() + + var numLocalpart int64 + numLocalpart, err = d.accounts.selectNewNumericLocalpart(ctx, txn) + if err != nil { + return err + } + localpart := strconv.FormatInt(numLocalpart, 10) + acc, err = d.createAccount(ctx, txn, localpart, "", "") + return err + }) + return acc, err +} + // CreateAccount makes a new account with the given login name and password, and creates an empty profile // for this account. If no password is supplied, the account will be a passwordless account. If the // account already exists, it will return nil, nil. func (d *Database) CreateAccount( ctx context.Context, localpart, plaintextPassword, appserviceID string, +) (acc *authtypes.Account, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + acc, err = d.createAccount(ctx, txn, localpart, plaintextPassword, appserviceID) + return err + }) + return +} + +func (d *Database) createAccount( + ctx context.Context, txn *sql.Tx, localpart, plaintextPassword, appserviceID string, ) (*authtypes.Account, error) { var err error - // Generate a password hash if this is not a password-less user hash := "" if plaintextPassword != "" { @@ -134,13 +170,14 @@ func (d *Database) CreateAccount( return nil, err } } - if err := d.profiles.insertProfile(ctx, localpart); err != nil { + if err := d.profiles.insertProfile(ctx, txn, localpart); err != nil { if common.IsUniqueConstraintViolationErr(err) { return nil, nil } return nil, err } - if err := d.SaveAccountData(ctx, localpart, "", "m.push_rules", `{ + + if err := d.accountDatas.insertAccountData(ctx, txn, localpart, "", "m.push_rules", `{ "global": { "content": [], "override": [], @@ -151,7 +188,7 @@ func (d *Database) CreateAccount( }`); err != nil { return nil, err } - return d.accounts.insertAccount(ctx, localpart, hash, appserviceID) + return d.accounts.insertAccount(ctx, txn, localpart, hash, appserviceID) } // SaveMembership saves the user matching a given localpart as a member of a given @@ -258,7 +295,9 @@ func (d *Database) newMembership( func (d *Database) SaveAccountData( ctx context.Context, localpart, roomID, dataType, content string, ) error { - return d.accountDatas.insertAccountData(ctx, localpart, roomID, dataType, content) + return common.WithTransaction(d.db, func(txn *sql.Tx) error { + return d.accountDatas.insertAccountData(ctx, txn, localpart, roomID, dataType, content) + }) } // GetAccountData returns account data related to a given localpart @@ -288,7 +327,7 @@ func (d *Database) GetAccountDataByType( func (d *Database) GetNewNumericLocalpart( ctx context.Context, ) (int64, error) { - return d.accounts.selectNewNumericLocalpart(ctx) + return d.accounts.selectNewNumericLocalpart(ctx, nil) } func hashPassword(plaintext string) (hash string, err error) { diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index ba24e5273..2de7b2733 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -516,16 +516,7 @@ func handleGuestRegistration( accountDB accounts.Database, deviceDB devices.Database, ) util.JSONResponse { - - //Generate numeric local part for guest user - id, err := accountDB.GetNewNumericLocalpart(req.Context()) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetNewNumericLocalpart failed") - return jsonerror.InternalServerError() - } - - localpart := strconv.FormatInt(id, 10) - acc, err := accountDB.CreateAccount(req.Context(), localpart, "", "") + acc, err := accountDB.CreateGuestAccount(req.Context()) if err != nil { return util.JSONResponse{ Code: http.StatusInternalServerError, From cdc115778551af69b4c9c39f758adcae61701aa3 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Mon, 9 Mar 2020 14:37:51 +0000 Subject: [PATCH 3/4] Improve logging when sending events (#883) We have some failing sytests on sqlite but it's very difficult to debug due to lack of useful logging. This adds a log line for when a new event is sent (incl. logging the event ID) as well as adding a user_id field for all contextual logs so we know who initiated certain actions. --- clientapi/routing/sendevent.go | 1 + common/httpapi.go | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index 47ad18825..da96c5c65 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -77,6 +77,7 @@ func SendEvent( util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed") return jsonerror.InternalServerError() } + util.GetLogger(req.Context()).WithField("event_id", eventID).Info("Sent event") res := util.JSONResponse{ Code: http.StatusOK, diff --git a/common/httpapi.go b/common/httpapi.go index 59b303b6d..22c774475 100644 --- a/common/httpapi.go +++ b/common/httpapi.go @@ -25,6 +25,10 @@ func MakeAuthAPI( if err != nil { return *err } + // add the user ID to the logger + logger := util.GetLogger((req.Context())) + logger = logger.WithField("user_id", device.UserID) + req = req.WithContext(util.ContextWithLogger(req.Context(), logger)) return f(req, device) } From 176f722d53fb47f68b77752aad56f03ac8c3ed4b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 10 Mar 2020 11:42:40 +0000 Subject: [PATCH 4/4] Update .gitignore --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index a36cb820a..1de8887ce 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,9 @@ _testmain.go # Default configuration file dendrite.yaml + +# Database files +*.db + +# Log files +*.log*