Merge branch 'master' into kegan/msc2836-2

This commit is contained in:
Kegsay 2020-12-02 15:03:15 +00:00 committed by GitHub
commit b68cd6fff5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 217 additions and 28 deletions

View file

@ -1,5 +1,21 @@
# Changelog
## Dendrite 0.3.1 (2020-11-20)
### Features
* Memory optimisation by reference passing, significantly reducing the number of allocations and duplication in memory
* A hook API has been added for experimental MSCs, with an early implementation of MSC2836
* The last seen timestamp and IP address are now updated automatically when calling `/sync`
* The last seen timestamp and IP address are now reported in `/_matrix/client/r0/devices` (contributed by [alexkursell](https://github.com/alexkursell))
* An optional configuration option `sync_api.real_ip_header` has been added for specifying which HTTP header contains the real client IP address (for if Dendrite is running behind a reverse HTTP proxy)
* Partial implementation of `/_matrix/client/r0/admin/whois` (contributed by [DavidSpenler](https://github.com/DavidSpenler))
### Fixes
* A concurrency bug has been fixed in the federation API that could cause Dendrite to crash
* The error when registering a username with invalid characters has been corrected (contributed by [bodqhrohro](https://github.com/bodqhrohro))
## Dendrite 0.3.0 (2020-11-16)
### Features

View file

@ -112,22 +112,24 @@ func (m *DendriteMonolith) Start() {
serverKeyAPI := &signing.YggdrasilKeys{}
keyRing := serverKeyAPI.KeyRing()
keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation)
userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI)
keyAPI.SetUserAPI(userAPI)
rsAPI := roomserver.NewInternalAPI(
base, keyRing,
)
fsAPI := federationsender.NewInternalAPI(
base, federation, rsAPI, keyRing,
)
keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation)
userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI)
keyAPI.SetUserAPI(userAPI)
eduInputAPI := eduserver.NewInternalAPI(
base, cache.New(), userAPI,
)
asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
fsAPI := federationsender.NewInternalAPI(
base, federation, rsAPI, keyRing,
)
ygg.SetSessionFunc(func(address string) {
req := &api.PerformServersAliveRequest{

View file

@ -16,6 +16,7 @@ package routing
import (
"io/ioutil"
"net"
"net/http"
"github.com/matrix-org/dendrite/clientapi/auth"
@ -32,7 +33,7 @@ type deviceJSON struct {
DeviceID string `json:"device_id"`
DisplayName string `json:"display_name"`
LastSeenIP string `json:"last_seen_ip"`
LastSeenTS uint64 `json:"last_seen_ts"`
LastSeenTS int64 `json:"last_seen_ts"`
}
type devicesJSON struct {
@ -79,6 +80,8 @@ func GetDeviceByID(
JSON: deviceJSON{
DeviceID: targetDevice.ID,
DisplayName: targetDevice.DisplayName,
LastSeenIP: stripIPPort(targetDevice.LastSeenIP),
LastSeenTS: targetDevice.LastSeenTS,
},
}
}
@ -102,6 +105,8 @@ func GetDevicesByLocalpart(
res.Devices = append(res.Devices, deviceJSON{
DeviceID: dev.ID,
DisplayName: dev.DisplayName,
LastSeenIP: stripIPPort(dev.LastSeenIP),
LastSeenTS: dev.LastSeenTS,
})
}
@ -230,3 +235,20 @@ func DeleteDevices(
JSON: struct{}{},
}
}
// stripIPPort converts strings like "[::1]:12345" to "::1"
func stripIPPort(addr string) string {
ip := net.ParseIP(addr)
if ip != nil {
return addr
}
host, _, err := net.SplitHostPort(addr)
if err != nil {
return ""
}
ip = net.ParseIP(host)
if ip != nil {
return host
}
return ""
}

View file

@ -21,7 +21,8 @@ import (
)
func KeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
intAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, base.CreateFederationClient())
fsAPI := base.FederationSenderHTTPClient()
intAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, fsAPI)
intAPI.SetUserAPI(base.UserAPIClient())
keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI)

View file

@ -300,6 +300,11 @@ sync_api:
max_idle_conns: 2
conn_max_lifetime: -1
# This option controls which HTTP header to inspect to find the real remote IP
# address of the client. This is likely required if Dendrite is running behind
# a reverse proxy server.
# real_ip_header: X-Real-IP
# Configuration for the User API.
user_api:
internal_api:

View file

@ -85,6 +85,7 @@ Set up the database:
```sh
sudo -u postgres psql -c "CREATE USER dendrite PASSWORD 'itsasecret'"
sudo -u postgres psql -c "ALTER USER dendrite CREATEDB"
for i in dendrite0 dendrite1 sytest_template; do sudo -u postgres psql -c "CREATE DATABASE $i OWNER dendrite;"; done
mkdir -p "server-0"
cat > "server-0/database.yaml" << EOF

View file

@ -7,6 +7,8 @@ type SyncAPI struct {
ExternalAPI ExternalAPIOptions `yaml:"external_api"`
Database DatabaseOptions `yaml:"database"`
RealIPHeader string `yaml:"real_ip_header"`
}
func (c *SyncAPI) Defaults() {

View file

@ -518,6 +518,9 @@ func (u *testUserAPI) PerformDeviceDeletion(ctx context.Context, req *userapi.Pe
func (u *testUserAPI) PerformDeviceUpdate(ctx context.Context, req *userapi.PerformDeviceUpdateRequest, res *userapi.PerformDeviceUpdateResponse) error {
return nil
}
func (u *testUserAPI) PerformLastSeenUpdate(ctx context.Context, req *userapi.PerformLastSeenUpdateRequest, res *userapi.PerformLastSeenUpdateResponse) error {
return nil
}
func (u *testUserAPI) PerformAccountDeactivation(ctx context.Context, req *userapi.PerformAccountDeactivationRequest, res *userapi.PerformAccountDeactivationResponse) error {
return nil
}

View file

@ -17,7 +17,7 @@ var build string
const (
VersionMajor = 0
VersionMinor = 3
VersionPatch = 0
VersionPatch = 1
VersionTag = "" // example: "rc1"
)

View file

@ -5,6 +5,7 @@ import (
"database/sql"
"errors"
"fmt"
"strings"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/roomserver/auth"
@ -222,12 +223,45 @@ func CheckServerAllowedToSeeEvent(
if errors.Is(err, sql.ErrNoRows) {
return false, nil
}
return false, err
return false, fmt.Errorf("roomState.LoadStateAtEvent: %w", err)
}
// TODO: We probably want to make it so that we don't have to pull
// out all the state if possible.
stateAtEvent, err := LoadStateEvents(ctx, db, stateEntries)
// Extract all of the event state key NIDs from the room state.
var stateKeyNIDs []types.EventStateKeyNID
for _, entry := range stateEntries {
stateKeyNIDs = append(stateKeyNIDs, entry.EventStateKeyNID)
}
// Then request those state key NIDs from the database.
stateKeys, err := db.EventStateKeys(ctx, stateKeyNIDs)
if err != nil {
return false, fmt.Errorf("db.EventStateKeys: %w", err)
}
// If the event state key doesn't match the given servername
// then we'll filter it out. This does preserve state keys that
// are "" since these will contain history visibility etc.
for nid, key := range stateKeys {
if key != "" && !strings.HasSuffix(key, ":"+string(serverName)) {
delete(stateKeys, nid)
}
}
// Now filter through all of the state events for the room.
// If the state key NID appears in the list of valid state
// keys then we'll add it to the list of filtered entries.
var filteredEntries []types.StateEntry
for _, entry := range stateEntries {
if _, ok := stateKeys[entry.EventStateKeyNID]; ok {
filteredEntries = append(filteredEntries, entry)
}
}
if len(filteredEntries) == 0 {
return false, nil
}
stateAtEvent, err := LoadStateEvents(ctx, db, filteredEntries)
if err != nil {
return false, err
}

View file

@ -19,10 +19,14 @@ package sync
import (
"context"
"fmt"
"net"
"net/http"
"strings"
"sync"
"time"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/config"
keyapi "github.com/matrix-org/dendrite/keyserver/api"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/syncapi/internal"
@ -37,18 +41,62 @@ import (
// RequestPool manages HTTP long-poll connections for /sync
type RequestPool struct {
db storage.Database
cfg *config.SyncAPI
userAPI userapi.UserInternalAPI
notifier *Notifier
keyAPI keyapi.KeyInternalAPI
rsAPI roomserverAPI.RoomserverInternalAPI
lastseen sync.Map
}
// NewRequestPool makes a new RequestPool
func NewRequestPool(
db storage.Database, n *Notifier, userAPI userapi.UserInternalAPI, keyAPI keyapi.KeyInternalAPI,
db storage.Database, cfg *config.SyncAPI, n *Notifier,
userAPI userapi.UserInternalAPI, keyAPI keyapi.KeyInternalAPI,
rsAPI roomserverAPI.RoomserverInternalAPI,
) *RequestPool {
return &RequestPool{db, userAPI, n, keyAPI, rsAPI}
rp := &RequestPool{db, cfg, userAPI, n, keyAPI, rsAPI, sync.Map{}}
go rp.cleanLastSeen()
return rp
}
func (rp *RequestPool) cleanLastSeen() {
for {
rp.lastseen.Range(func(key interface{}, _ interface{}) bool {
rp.lastseen.Delete(key)
return true
})
time.Sleep(time.Minute)
}
}
func (rp *RequestPool) updateLastSeen(req *http.Request, device *userapi.Device) {
if _, ok := rp.lastseen.LoadOrStore(device.UserID+device.ID, struct{}{}); ok {
return
}
remoteAddr := req.RemoteAddr
if rp.cfg.RealIPHeader != "" {
if header := req.Header.Get(rp.cfg.RealIPHeader); header != "" {
// TODO: Maybe this isn't great but it will satisfy both X-Real-IP
// and X-Forwarded-For (which can be a list where the real client
// address is the first listed address). Make more intelligent?
addresses := strings.Split(header, ",")
if ip := net.ParseIP(addresses[0]); ip != nil {
remoteAddr = addresses[0]
}
}
}
lsreq := &userapi.PerformLastSeenUpdateRequest{
UserID: device.UserID,
DeviceID: device.ID,
RemoteAddr: remoteAddr,
}
lsres := &userapi.PerformLastSeenUpdateResponse{}
go rp.userAPI.PerformLastSeenUpdate(req.Context(), lsreq, lsres) // nolint:errcheck
rp.lastseen.Store(device.UserID+device.ID, time.Now())
}
// OnIncomingSyncRequest is called when a client makes a /sync request. This function MUST be
@ -74,6 +122,8 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
"limit": syncReq.limit,
})
rp.updateLastSeen(req, device)
currPos := rp.notifier.CurrentPosition()
if rp.shouldReturnImmediately(syncReq) {
@ -278,7 +328,7 @@ func (rp *RequestPool) appendAccountData(
// data keys were set between two message. This isn't a huge issue since the
// duplicate data doesn't represent a huge quantity of data, but an optimisation
// here would be making sure each data is sent only once to the client.
if req.since == nil {
if req.since == nil || (req.since.PDUPosition() == 0 && req.since.EDUPosition() == 0) {
// If this is the initial sync, we don't need to check if a data has
// already been sent. Instead, we send the whole batch.
dataReq := &userapi.QueryAccountDataRequest{

View file

@ -61,7 +61,7 @@ func AddPublicRoutes(
logrus.WithError(err).Panicf("failed to start notifier")
}
requestPool := sync.NewRequestPool(syncDB, notifier, userAPI, keyAPI, rsAPI)
requestPool := sync.NewRequestPool(syncDB, cfg, notifier, userAPI, keyAPI, rsAPI)
keyChangeConsumer := consumers.NewOutputKeyChangeEventConsumer(
cfg.Matrix.ServerName, string(cfg.Matrix.Kafka.TopicFor(config.TopicOutputKeyChangeEvent)),

View file

@ -29,6 +29,7 @@ type UserInternalAPI interface {
PerformPasswordUpdate(ctx context.Context, req *PerformPasswordUpdateRequest, res *PerformPasswordUpdateResponse) error
PerformDeviceCreation(ctx context.Context, req *PerformDeviceCreationRequest, res *PerformDeviceCreationResponse) error
PerformDeviceDeletion(ctx context.Context, req *PerformDeviceDeletionRequest, res *PerformDeviceDeletionResponse) error
PerformLastSeenUpdate(ctx context.Context, req *PerformLastSeenUpdateRequest, res *PerformLastSeenUpdateResponse) error
PerformDeviceUpdate(ctx context.Context, req *PerformDeviceUpdateRequest, res *PerformDeviceUpdateResponse) error
PerformAccountDeactivation(ctx context.Context, req *PerformAccountDeactivationRequest, res *PerformAccountDeactivationResponse) error
QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error
@ -183,6 +184,17 @@ type PerformPasswordUpdateResponse struct {
Account *Account
}
// PerformLastSeenUpdateRequest is the request for PerformLastSeenUpdate.
type PerformLastSeenUpdateRequest struct {
UserID string
DeviceID string
RemoteAddr string
}
// PerformLastSeenUpdateResponse is the response for PerformLastSeenUpdate.
type PerformLastSeenUpdateResponse struct {
}
// PerformDeviceCreationRequest is the request for PerformDeviceCreation
type PerformDeviceCreationRequest struct {
Localpart string

View file

@ -172,6 +172,21 @@ func (a *UserInternalAPI) deviceListUpdate(userID string, deviceIDs []string) er
return nil
}
func (a *UserInternalAPI) PerformLastSeenUpdate(
ctx context.Context,
req *api.PerformLastSeenUpdateRequest,
res *api.PerformLastSeenUpdateResponse,
) error {
localpart, _, err := gomatrixserverlib.SplitID('@', req.UserID)
if err != nil {
return fmt.Errorf("gomatrixserverlib.SplitID: %w", err)
}
if err := a.DeviceDB.UpdateDeviceLastSeen(ctx, localpart, req.DeviceID, req.RemoteAddr); err != nil {
return fmt.Errorf("a.DeviceDB.UpdateDeviceLastSeen: %w", err)
}
return nil
}
func (a *UserInternalAPI) PerformDeviceUpdate(ctx context.Context, req *api.PerformDeviceUpdateRequest, res *api.PerformDeviceUpdateResponse) error {
localpart, _, err := gomatrixserverlib.SplitID('@', req.RequestingUserID)
if err != nil {

View file

@ -32,6 +32,7 @@ const (
PerformAccountCreationPath = "/userapi/performAccountCreation"
PerformPasswordUpdatePath = "/userapi/performPasswordUpdate"
PerformDeviceDeletionPath = "/userapi/performDeviceDeletion"
PerformLastSeenUpdatePath = "/userapi/performLastSeenUpdate"
PerformDeviceUpdatePath = "/userapi/performDeviceUpdate"
PerformAccountDeactivationPath = "/userapi/performAccountDeactivation"
@ -119,6 +120,18 @@ func (h *httpUserInternalAPI) PerformDeviceDeletion(
return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
}
func (h *httpUserInternalAPI) PerformLastSeenUpdate(
ctx context.Context,
req *api.PerformLastSeenUpdateRequest,
res *api.PerformLastSeenUpdateResponse,
) error {
span, ctx := opentracing.StartSpanFromContext(ctx, "PerformLastSeen")
defer span.Finish()
apiURL := h.apiURL + PerformLastSeenUpdatePath
return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
}
func (h *httpUserInternalAPI) PerformDeviceUpdate(ctx context.Context, req *api.PerformDeviceUpdateRequest, res *api.PerformDeviceUpdateResponse) error {
span, ctx := opentracing.StartSpanFromContext(ctx, "PerformDeviceUpdate")
defer span.Finish()

View file

@ -65,6 +65,19 @@ func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) {
return util.JSONResponse{Code: http.StatusOK, JSON: &response}
}),
)
internalAPIMux.Handle(PerformLastSeenUpdatePath,
httputil.MakeInternalAPI("performLastSeenUpdate", func(req *http.Request) util.JSONResponse {
request := api.PerformLastSeenUpdateRequest{}
response := api.PerformLastSeenUpdateResponse{}
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
return util.MessageResponse(http.StatusBadRequest, err.Error())
}
if err := s.PerformLastSeenUpdate(req.Context(), &request, &response); err != nil {
return util.ErrorResponse(err)
}
return util.JSONResponse{Code: http.StatusOK, JSON: &response}
}),
)
internalAPIMux.Handle(PerformDeviceUpdatePath,
httputil.MakeInternalAPI("performDeviceUpdate", func(req *http.Request) util.JSONResponse {
request := api.PerformDeviceUpdateRequest{}

View file

@ -33,9 +33,9 @@ type Database interface {
// Returns the device on success.
CreateDevice(ctx context.Context, localpart string, deviceID *string, accessToken string, displayName *string, ipAddr, userAgent string) (dev *api.Device, returnErr error)
UpdateDevice(ctx context.Context, localpart, deviceID string, displayName *string) error
UpdateDeviceLastSeen(ctx context.Context, localpart, deviceID, ipAddr string) error
RemoveDevice(ctx context.Context, deviceID, localpart string) error
RemoveDevices(ctx context.Context, localpart string, devices []string) error
// RemoveAllDevices deleted all devices for this user. Returns the devices deleted.
RemoveAllDevices(ctx context.Context, localpart, exceptDeviceID string) (devices []api.Device, err error)
UpdateDeviceLastSeen(ctx context.Context, deviceID, ipAddr string) error
}

View file

@ -95,7 +95,7 @@ const selectDevicesByIDSQL = "" +
"SELECT device_id, localpart, display_name FROM device_devices WHERE device_id = ANY($1)"
const updateDeviceLastSeen = "" +
"UPDATE device_devices SET last_seen_ts = $1, ip = $2 WHERE device_id = $3"
"UPDATE device_devices SET last_seen_ts = $1, ip = $2 WHERE localpart = $3 AND device_id = $4"
type devicesStatements struct {
insertDeviceStmt *sql.Stmt
@ -310,9 +310,9 @@ func (s *devicesStatements) selectDevicesByLocalpart(
return devices, rows.Err()
}
func (s *devicesStatements) updateDeviceLastSeen(ctx context.Context, txn *sql.Tx, deviceID, ipAddr string) error {
func (s *devicesStatements) updateDeviceLastSeen(ctx context.Context, txn *sql.Tx, localpart, deviceID, ipAddr string) error {
lastSeenTs := time.Now().UnixNano() / 1000000
stmt := sqlutil.TxStmt(txn, s.updateDeviceLastSeenStmt)
_, err := stmt.ExecContext(ctx, lastSeenTs, ipAddr, deviceID)
_, err := stmt.ExecContext(ctx, lastSeenTs, ipAddr, localpart, deviceID)
return err
}

View file

@ -205,8 +205,8 @@ func (d *Database) RemoveAllDevices(
}
// UpdateDeviceLastSeen updates a the last seen timestamp and the ip address
func (d *Database) UpdateDeviceLastSeen(ctx context.Context, deviceID, ipAddr string) error {
func (d *Database) UpdateDeviceLastSeen(ctx context.Context, localpart, deviceID, ipAddr string) error {
return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error {
return d.devices.updateDeviceLastSeen(ctx, txn, deviceID, ipAddr)
return d.devices.updateDeviceLastSeen(ctx, txn, localpart, deviceID, ipAddr)
})
}

View file

@ -80,7 +80,7 @@ const selectDevicesByIDSQL = "" +
"SELECT device_id, localpart, display_name FROM device_devices WHERE device_id IN ($1)"
const updateDeviceLastSeen = "" +
"UPDATE device_devices SET last_seen_ts = $1, ip = $2 WHERE device_id = $3"
"UPDATE device_devices SET last_seen_ts = $1, ip = $2 WHERE localpart = $3 AND device_id = $4"
type devicesStatements struct {
db *sql.DB
@ -314,9 +314,9 @@ func (s *devicesStatements) selectDevicesByID(ctx context.Context, deviceIDs []s
return devices, rows.Err()
}
func (s *devicesStatements) updateDeviceLastSeen(ctx context.Context, txn *sql.Tx, deviceID, ipAddr string) error {
func (s *devicesStatements) updateDeviceLastSeen(ctx context.Context, txn *sql.Tx, localpart, deviceID, ipAddr string) error {
lastSeenTs := time.Now().UnixNano() / 1000000
stmt := sqlutil.TxStmt(txn, s.updateDeviceLastSeenStmt)
_, err := stmt.ExecContext(ctx, lastSeenTs, ipAddr, deviceID)
_, err := stmt.ExecContext(ctx, lastSeenTs, ipAddr, localpart, deviceID)
return err
}

View file

@ -207,8 +207,8 @@ func (d *Database) RemoveAllDevices(
}
// UpdateDeviceLastSeen updates a the last seen timestamp and the ip address
func (d *Database) UpdateDeviceLastSeen(ctx context.Context, deviceID, ipAddr string) error {
func (d *Database) UpdateDeviceLastSeen(ctx context.Context, localpart, deviceID, ipAddr string) error {
return d.writer.Do(d.db, nil, func(txn *sql.Tx) error {
return d.devices.updateDeviceLastSeen(ctx, txn, deviceID, ipAddr)
return d.devices.updateDeviceLastSeen(ctx, txn, localpart, deviceID, ipAddr)
})
}