From 7f26b0cd13534ba235c2bf0a7220d3462ca8e9da Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 19 Jun 2020 13:29:27 +0100 Subject: [PATCH] Bind build support, further Yggdrasil demo updates (#1152) * Add gobind builds for Yggdrasil demo * Massage client API a bit * Fix build * Fix gobind build * Fix gobind client API setup * Tweaks * Tweaks * Update sytest-whitelist, add comment * Default to sending push rules on initial sync --- build/gobind/build.sh | 6 + build/gobind/monolith.go | 161 ++++++++++++++++++ build/gobind/platform_ios.go | 25 +++ build/gobind/platform_other.go | 12 ++ clientapi/routing/device.go | 9 +- clientapi/routing/joinroom.go | 1 + cmd/dendrite-demo-yggdrasil/main.go | 49 +----- cmd/dendrite-demo-yggdrasil/yggconn/client.go | 74 ++++++++ cmd/dendrite-demo-yggdrasil/yggconn/node.go | 27 +-- go.mod | 1 + go.sum | 15 ++ syncapi/sync/request.go | 4 + syncapi/sync/requestpool.go | 3 +- syncapi/types/types.go | 11 +- sytest-whitelist | 3 + 15 files changed, 336 insertions(+), 65 deletions(-) create mode 100644 build/gobind/build.sh create mode 100644 build/gobind/monolith.go create mode 100644 build/gobind/platform_ios.go create mode 100644 build/gobind/platform_other.go create mode 100644 cmd/dendrite-demo-yggdrasil/yggconn/client.go diff --git a/build/gobind/build.sh b/build/gobind/build.sh new file mode 100644 index 000000000..3a80d374a --- /dev/null +++ b/build/gobind/build.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +gomobile bind -v \ + -ldflags "-X $github.com/yggdrasil-network/yggdrasil-go/src/version.buildName=riot-ios-p2p" \ + -target ios \ + github.com/matrix-org/dendrite/build/gobind \ No newline at end of file diff --git a/build/gobind/monolith.go b/build/gobind/monolith.go new file mode 100644 index 000000000..750babad8 --- /dev/null +++ b/build/gobind/monolith.go @@ -0,0 +1,161 @@ +package gobind + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "net/http" + "time" + + "github.com/matrix-org/dendrite/appservice" + "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" + "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn" + "github.com/matrix-org/dendrite/eduserver" + "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/federationsender" + "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/setup" + "github.com/matrix-org/dendrite/publicroomsapi/storage" + "github.com/matrix-org/dendrite/roomserver" + "github.com/matrix-org/dendrite/userapi" + "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" +) + +type DendriteMonolith struct { + StorageDirectory string + listener net.Listener +} + +func (m *DendriteMonolith) BaseURL() string { + return fmt.Sprintf("http://%s", m.listener.Addr().String()) +} + +func (m *DendriteMonolith) Start() { + logger := logrus.Logger{ + Out: BindLogger{}, + } + logrus.SetOutput(BindLogger{}) + + var err error + m.listener, err = net.Listen("tcp", "localhost:65432") + if err != nil { + panic(err) + } + + ygg, err := yggconn.Setup("dendrite", "", m.StorageDirectory) + if err != nil { + panic(err) + } + + cfg := &config.Dendrite{} + cfg.SetDefaults() + cfg.Matrix.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName()) + cfg.Matrix.PrivateKey = ygg.SigningPrivateKey() + cfg.Matrix.KeyID = gomatrixserverlib.KeyID(signing.KeyID) + cfg.Kafka.UseNaffka = true + cfg.Kafka.Topics.OutputRoomEvent = "roomserverOutput" + cfg.Kafka.Topics.OutputClientData = "clientapiOutput" + cfg.Kafka.Topics.OutputTypingEvent = "typingServerOutput" + cfg.Kafka.Topics.OutputSendToDeviceEvent = "sendToDeviceOutput" + cfg.Database.Account = config.DataSource(fmt.Sprintf("file:%s/dendrite-account.db", m.StorageDirectory)) + cfg.Database.Device = config.DataSource(fmt.Sprintf("file:%s/dendrite-device.db", m.StorageDirectory)) + cfg.Database.MediaAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-mediaapi.db", m.StorageDirectory)) + cfg.Database.SyncAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-syncapi.db", m.StorageDirectory)) + cfg.Database.RoomServer = config.DataSource(fmt.Sprintf("file:%s/dendrite-roomserver.db", m.StorageDirectory)) + cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s/dendrite-serverkey.db", m.StorageDirectory)) + cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s/dendrite-federationsender.db", m.StorageDirectory)) + cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s/dendrite-appservice.db", m.StorageDirectory)) + cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-publicroomsa.db", m.StorageDirectory)) + cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s/dendrite-naffka.db", m.StorageDirectory)) + if err = cfg.Derive(); err != nil { + panic(err) + } + + base := setup.NewBaseDendrite(cfg, "Monolith", false) + defer base.Close() // nolint: errcheck + + accountDB := base.CreateAccountsDB() + deviceDB := base.CreateDeviceDB() + federation := ygg.CreateFederationClient(base) + + serverKeyAPI := &signing.YggdrasilKeys{} + keyRing := serverKeyAPI.KeyRing() + userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, cfg.Derived.ApplicationServices) + + rsAPI := roomserver.NewInternalAPI( + base, keyRing, federation, + ) + + eduInputAPI := eduserver.NewInternalAPI( + base, cache.New(), userAPI, + ) + + asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) + + fsAPI := federationsender.NewInternalAPI( + base, federation, rsAPI, keyRing, + ) + + // The underlying roomserver implementation needs to be able to call the fedsender. + // This is different to rsAPI which can be the http client which doesn't need this dependency + rsAPI.SetFederationSenderAPI(fsAPI) + + publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName) + if err != nil { + logrus.WithError(err).Panicf("failed to connect to public rooms db") + } + + monolith := setup.Monolith{ + Config: base.Cfg, + AccountDB: accountDB, + DeviceDB: deviceDB, + Client: ygg.CreateClient(base), + FedClient: federation, + KeyRing: keyRing, + KafkaConsumer: base.KafkaConsumer, + KafkaProducer: base.KafkaProducer, + + AppserviceAPI: asAPI, + EDUInternalAPI: eduInputAPI, + FederationSenderAPI: fsAPI, + RoomserverAPI: rsAPI, + UserAPI: userAPI, + //ServerKeyAPI: serverKeyAPI, + + PublicRoomsDB: publicRoomsDB, + } + monolith.AddAllPublicRoutes(base.PublicAPIMux) + + httputil.SetupHTTPAPI( + base.BaseMux, + base.PublicAPIMux, + base.InternalAPIMux, + cfg, + base.UseHTTPAPIs, + ) + + // Build both ends of a HTTP multiplex. + httpServer := &http.Server{ + Addr: ":0", + TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){}, + ReadTimeout: 15 * time.Second, + WriteTimeout: 45 * time.Second, + IdleTimeout: 60 * time.Second, + BaseContext: func(_ net.Listener) context.Context { + return context.Background() + }, + Handler: base.BaseMux, + } + + go func() { + logger.Info("Listening on ", ygg.DerivedServerName()) + logger.Fatal(httpServer.Serve(ygg)) + }() + go func() { + logger.Info("Listening on ", m.BaseURL()) + logger.Fatal(httpServer.Serve(m.listener)) + }() +} diff --git a/build/gobind/platform_ios.go b/build/gobind/platform_ios.go new file mode 100644 index 000000000..01f8a6a04 --- /dev/null +++ b/build/gobind/platform_ios.go @@ -0,0 +1,25 @@ +// +build ios + +package gobind + +/* +#cgo CFLAGS: -x objective-c +#cgo LDFLAGS: -framework Foundation +#import +void Log(const char *text) { + NSString *nss = [NSString stringWithUTF8String:text]; + NSLog(@"%@", nss); +} +*/ +import "C" +import "unsafe" + +type BindLogger struct { +} + +func (nsl BindLogger) Write(p []byte) (n int, err error) { + p = append(p, 0) + cstr := (*C.char)(unsafe.Pointer(&p[0])) + C.Log(cstr) + return len(p), nil +} diff --git a/build/gobind/platform_other.go b/build/gobind/platform_other.go new file mode 100644 index 000000000..fdfb13bc0 --- /dev/null +++ b/build/gobind/platform_other.go @@ -0,0 +1,12 @@ +// +build !ios + +package gobind + +import "log" + +type BindLogger struct{} + +func (nsl BindLogger) Write(p []byte) (n int, err error) { + log.Println(string(p)) + return len(p), nil +} diff --git a/clientapi/routing/device.go b/clientapi/routing/device.go index 403937c9c..51a15a882 100644 --- a/clientapi/routing/device.go +++ b/clientapi/routing/device.go @@ -26,9 +26,12 @@ import ( "github.com/matrix-org/util" ) +// https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-devices type deviceJSON struct { - DeviceID string `json:"device_id"` - UserID string `json:"user_id"` + DeviceID string `json:"device_id"` + DisplayName string `json:"display_name"` + LastSeenIP string `json:"last_seen_ip"` + LastSeenTS uint64 `json:"last_seen_ts"` } type devicesJSON struct { @@ -70,7 +73,6 @@ func GetDeviceByID( Code: http.StatusOK, JSON: deviceJSON{ DeviceID: dev.ID, - UserID: dev.UserID, }, } } @@ -98,7 +100,6 @@ func GetDevicesByLocalpart( for _, dev := range deviceList { res.Devices = append(res.Devices, deviceJSON{ DeviceID: dev.ID, - UserID: dev.UserID, }) } diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index 3871e4d4e..db890d03f 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -37,6 +37,7 @@ func JoinRoomByIDOrAlias( joinReq := roomserverAPI.PerformJoinRequest{ RoomIDOrAlias: roomIDOrAlias, UserID: device.UserID, + Content: map[string]interface{}{}, } joinRes := roomserverAPI.PerformJoinResponse{} diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 328aa0629..db05ecb76 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -47,52 +47,11 @@ var ( instancePeer = flag.String("peer", "", "an internet Yggdrasil peer to connect to") ) -type yggroundtripper struct { - inner *http.Transport -} - -func (y *yggroundtripper) RoundTrip(req *http.Request) (*http.Response, error) { - req.URL.Scheme = "http" - return y.inner.RoundTrip(req) -} - -func createFederationClient( - base *setup.BaseDendrite, n *yggconn.Node, -) *gomatrixserverlib.FederationClient { - tr := &http.Transport{} - tr.RegisterProtocol( - "matrix", &yggroundtripper{ - inner: &http.Transport{ - ResponseHeaderTimeout: 15 * time.Second, - IdleConnTimeout: 60 * time.Second, - DialContext: n.DialerContext, - }, - }, - ) - return gomatrixserverlib.NewFederationClientWithTransport( - base.Cfg.Matrix.ServerName, base.Cfg.Matrix.KeyID, base.Cfg.Matrix.PrivateKey, tr, - ) -} - -func createClient(n *yggconn.Node) *gomatrixserverlib.Client { - tr := &http.Transport{} - tr.RegisterProtocol( - "matrix", &yggroundtripper{ - inner: &http.Transport{ - ResponseHeaderTimeout: 15 * time.Second, - IdleConnTimeout: 60 * time.Second, - DialContext: n.DialerContext, - }, - }, - ) - return gomatrixserverlib.NewClientWithTransport(tr) -} - // nolint:gocyclo func main() { flag.Parse() - ygg, err := yggconn.Setup(*instanceName, *instancePeer) + ygg, err := yggconn.Setup(*instanceName, *instancePeer, ".") if err != nil { panic(err) } @@ -125,7 +84,7 @@ func main() { accountDB := base.CreateAccountsDB() deviceDB := base.CreateDeviceDB() - federation := createFederationClient(base, ygg) + federation := ygg.CreateFederationClient(base) serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() @@ -160,7 +119,7 @@ func main() { Config: base.Cfg, AccountDB: accountDB, DeviceDB: deviceDB, - Client: createClient(ygg), + Client: ygg.CreateClient(base), FedClient: federation, KeyRing: keyRing, KafkaConsumer: base.KafkaConsumer, @@ -203,7 +162,7 @@ func main() { logrus.Fatal(httpServer.Serve(ygg)) }() go func() { - httpBindAddr := fmt.Sprintf("localhost:%d", *instancePort) + httpBindAddr := fmt.Sprintf(":%d", *instancePort) logrus.Info("Listening on ", httpBindAddr) logrus.Fatal(http.ListenAndServe(httpBindAddr, base.BaseMux)) }() diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/client.go b/cmd/dendrite-demo-yggdrasil/yggconn/client.go new file mode 100644 index 000000000..399993e3e --- /dev/null +++ b/cmd/dendrite-demo-yggdrasil/yggconn/client.go @@ -0,0 +1,74 @@ +package yggconn + +import ( + "context" + "crypto/ed25519" + "encoding/hex" + "fmt" + "net" + "net/http" + "strings" + "time" + + "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/convert" + "github.com/matrix-org/dendrite/internal/setup" + "github.com/matrix-org/gomatrixserverlib" +) + +func (n *Node) yggdialer(_, address string) (net.Conn, error) { + tokens := strings.Split(address, ":") + raw, err := hex.DecodeString(tokens[0]) + if err != nil { + return nil, fmt.Errorf("hex.DecodeString: %w", err) + } + converted := convert.Ed25519PublicKeyToCurve25519(ed25519.PublicKey(raw)) + convhex := hex.EncodeToString(converted) + return n.Dial("curve25519", convhex) +} + +func (n *Node) yggdialerctx(ctx context.Context, network, address string) (net.Conn, error) { + return n.yggdialer(network, address) +} + +type yggroundtripper struct { + inner *http.Transport +} + +func (y *yggroundtripper) RoundTrip(req *http.Request) (*http.Response, error) { + req.URL.Scheme = "http" + return y.inner.RoundTrip(req) +} + +func (n *Node) CreateClient( + base *setup.BaseDendrite, +) *gomatrixserverlib.Client { + tr := &http.Transport{} + tr.RegisterProtocol( + "matrix", &yggroundtripper{ + inner: &http.Transport{ + ResponseHeaderTimeout: 15 * time.Second, + IdleConnTimeout: 60 * time.Second, + DialContext: n.yggdialerctx, + }, + }, + ) + return gomatrixserverlib.NewClientWithTransport(tr) +} + +func (n *Node) CreateFederationClient( + base *setup.BaseDendrite, +) *gomatrixserverlib.FederationClient { + tr := &http.Transport{} + tr.RegisterProtocol( + "matrix", &yggroundtripper{ + inner: &http.Transport{ + ResponseHeaderTimeout: 15 * time.Second, + IdleConnTimeout: 60 * time.Second, + DialContext: n.yggdialerctx, + }, + }, + ) + return gomatrixserverlib.NewFederationClientWithTransport( + base.Cfg.Matrix.ServerName, base.Cfg.Matrix.KeyID, base.Cfg.Matrix.PrivateKey, tr, + ) +} diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/node.go b/cmd/dendrite-demo-yggdrasil/yggconn/node.go index b225e1cf9..c335f2eac 100644 --- a/cmd/dendrite-demo-yggdrasil/yggconn/node.go +++ b/cmd/dendrite-demo-yggdrasil/yggconn/node.go @@ -67,7 +67,7 @@ func (n *Node) DialerContext(ctx context.Context, network, address string) (net. } // nolint:gocyclo -func Setup(instanceName, instancePeer string) (*Node, error) { +func Setup(instanceName, instancePeer, storageDirectory string) (*Node, error) { n := &Node{ core: &yggdrasil.Core{}, config: yggdrasilconfig.GenerateConfig(), @@ -77,7 +77,7 @@ func Setup(instanceName, instancePeer string) (*Node, error) { incoming: make(chan *yamux.Stream), } - yggfile := fmt.Sprintf("%s-yggdrasil.conf", instanceName) + yggfile := fmt.Sprintf("%s/%s-yggdrasil.conf", storageDirectory, instanceName) if _, err := os.Stat(yggfile); !os.IsNotExist(err) { yggconf, e := ioutil.ReadFile(yggfile) if e != nil { @@ -87,7 +87,7 @@ func Setup(instanceName, instancePeer string) (*Node, error) { panic(err) } } else { - n.config.AdminListen = fmt.Sprintf("unix://./%s-yggdrasil.sock", instanceName) + n.config.AdminListen = "none" // fmt.Sprintf("unix://%s/%s-yggdrasil.sock", storageDirectory, instanceName) n.config.MulticastInterfaces = []string{".*"} n.config.EncryptionPrivateKey = hex.EncodeToString(n.EncryptionPrivateKey()) n.config.EncryptionPublicKey = hex.EncodeToString(n.EncryptionPublicKey()) @@ -114,20 +114,22 @@ func Setup(instanceName, instancePeer string) (*Node, error) { panic(err) } } - if err = n.admin.Init(n.core, n.state, n.log, nil); err != nil { - panic(err) - } - if err = n.admin.Start(); err != nil { - panic(err) - } + /* + if err = n.admin.Init(n.core, n.state, n.log, nil); err != nil { + panic(err) + } + if err = n.admin.Start(); err != nil { + panic(err) + } + */ if err = n.multicast.Init(n.core, n.state, n.log, nil); err != nil { panic(err) } if err = n.multicast.Start(); err != nil { panic(err) } - n.admin.SetupAdminHandlers(n.admin) - n.multicast.SetupAdminHandlers(n.admin) + //n.admin.SetupAdminHandlers(n.admin) + //n.multicast.SetupAdminHandlers(n.admin) n.listener, err = n.core.ConnListen() if err != nil { panic(err) @@ -137,6 +139,9 @@ func Setup(instanceName, instancePeer string) (*Node, error) { panic(err) } + n.log.Println("Public curve25519:", n.core.EncryptionPublicKey()) + n.log.Println("Public ed25519:", n.core.SigningPublicKey()) + go n.listenFromYgg() return n, nil diff --git a/go.mod b/go.mod index ff73ad078..4473dc97f 100644 --- a/go.mod +++ b/go.mod @@ -38,6 +38,7 @@ require ( github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b go.uber.org/atomic v1.4.0 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d + golang.org/x/mobile v0.0.0-20200329125638-4c31acba0007 // indirect gopkg.in/h2non/bimg.v1 v1.0.18 gopkg.in/yaml.v2 v2.2.8 ) diff --git a/go.sum b/go.sum index f62b14b62..97abb7b98 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,7 @@ github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOv github.com/Arceliar/phony v0.0.0-20191006174943-d0c68492aca0 h1:p3puK8Sl2xK+2FnnIvY/C0N1aqJo2kbEsdAzU+Tnv48= github.com/Arceliar/phony v0.0.0-20191006174943-d0c68492aca0/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/Shopify/sarama v1.26.1 h1:3jnfWKD7gVwbB1KSy/lE0szA9duPuSFLViK0o/d3DgA= github.com/Shopify/sarama v1.26.1/go.mod h1:NbSGBSSndYaIhRcBtY9V0U7AyH+x71bG668AuWys/yU= @@ -589,6 +590,7 @@ golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -599,9 +601,18 @@ golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20200329125638-4c31acba0007 h1:JxsyO7zPDWn1rBZW8FV5RFwCKqYeXnyaS/VQPLpXu6I= +golang.org/x/mobile v0.0.0-20200329125638-4c31acba0007/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -671,6 +682,10 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69 h1:yBHHx+XZqXJBm6Exke3N7V9gnlsyXxoCPEb1yVenjfk= +golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= diff --git a/syncapi/sync/request.go b/syncapi/sync/request.go index beeaa40f7..5dd92c853 100644 --- a/syncapi/sync/request.go +++ b/syncapi/sync/request.go @@ -62,6 +62,10 @@ func newSyncRequest(req *http.Request, device userapi.Device) (*syncRequest, err } since = &tok } + if since == nil { + tok := types.NewStreamToken(0, 0) + since = &tok + } timelineLimit := defaultTimelineLimit // TODO: read from stored filters too filterQuery := req.URL.Query().Get("filter") diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 8d51689e3..743c63a62 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -256,7 +256,8 @@ func (rp *RequestPool) appendAccountData( } if len(dataTypes) == 0 { - return data, nil + // TODO: this fixes the sytest but is it the right thing to do? + dataTypes[""] = []string{"m.push_rules"} } // Iterate over the rooms diff --git a/syncapi/types/types.go b/syncapi/types/types.go index c1f09fba5..1094416a1 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -98,6 +98,9 @@ func (t *StreamingToken) PDUPosition() StreamPosition { func (t *StreamingToken) EDUPosition() StreamPosition { return t.Positions[1] } +func (t *StreamingToken) String() string { + return t.syncToken.String() +} // IsAfter returns true if ANY position in this token is greater than `other`. func (t *StreamingToken) IsAfter(other StreamingToken) bool { @@ -220,8 +223,8 @@ func NewTopologyTokenFromString(tok string) (token TopologyToken, err error) { err = fmt.Errorf("token %s is not a topology token", tok) return } - if len(t.Positions) != 2 { - err = fmt.Errorf("token %s wrong number of values, got %d want 2", tok, len(t.Positions)) + if len(t.Positions) < 2 { + err = fmt.Errorf("token %s wrong number of values, got %d want at least 2", tok, len(t.Positions)) return } return TopologyToken{ @@ -247,8 +250,8 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) { err = fmt.Errorf("token %s is not a streaming token", tok) return } - if len(t.Positions) != 2 { - err = fmt.Errorf("token %s wrong number of values, got %d want 2", tok, len(t.Positions)) + if len(t.Positions) < 2 { + err = fmt.Errorf("token %s wrong number of values, got %d want at least 2", tok, len(t.Positions)) return } return StreamingToken{ diff --git a/sytest-whitelist b/sytest-whitelist index 971f8cf0d..7edc79e64 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -347,3 +347,6 @@ GET /rooms/:room_id/state/m.room.topic gets topic GET /rooms/:room_id/state fetches entire room state Setting room topic reports m.room.topic to myself setting 'm.room.name' respects room powerlevel +Syncing a new room with a large timeline limit isn't limited +Left rooms appear in the leave section of sync +Banned rooms appear in the leave section of sync