From 98a5e410d7ecae49f525ddabc55a86d8a6731f22 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 23 Mar 2022 10:20:18 +0000 Subject: [PATCH 01/55] Per-room consumers (#2293) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Roomserver input refactoring — again! * Ensure the actor runs again * Preserve consumer after unsubscribe * Another sprinkling of magic * Rename `TopicFor` to `Prefixed` * Recreate the stream if the config is bad * Check streams too * Prefix subjects, preserve inboxes * Recreate if subjects wrong * Remove stream subject * Reconstruct properly * Fix mutex unlock * Comments * Fix tests * Don't drop events * Review comments * Separate `queueInputRoomEvents` function * Re-jig control flow a bit --- appservice/consumers/roomserver.go | 2 +- clientapi/clientapi.go | 2 +- eduserver/eduserver.go | 6 +- federationapi/consumers/eduserver.go | 6 +- federationapi/consumers/keychange.go | 4 +- federationapi/consumers/roomserver.go | 2 +- keyserver/keyserver.go | 2 +- roomserver/internal/api.go | 1 + roomserver/internal/input/input.go | 393 ++++++++++++++------ roomserver/roomserver.go | 4 +- setup/config/config_jetstream.go | 4 +- setup/jetstream/nats.go | 31 +- setup/jetstream/streams.go | 14 +- syncapi/consumers/clientapi.go | 2 +- syncapi/consumers/eduserver_receipts.go | 2 +- syncapi/consumers/eduserver_sendtodevice.go | 2 +- syncapi/consumers/eduserver_typing.go | 2 +- syncapi/consumers/roomserver.go | 2 +- syncapi/consumers/userapi.go | 2 +- syncapi/syncapi.go | 6 +- userapi/consumers/syncapi_readupdate.go | 2 +- userapi/consumers/syncapi_streamevent.go | 2 +- userapi/userapi.go | 4 +- 23 files changed, 345 insertions(+), 152 deletions(-) diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index 9d723bed1..01790722a 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -56,7 +56,7 @@ func NewOutputRoomEventConsumer( ctx: process.Context(), jetstream: js, durable: cfg.Global.JetStream.Durable("AppserviceRoomserverConsumer"), - topic: cfg.Global.JetStream.TopicFor(jetstream.OutputRoomEvent), + topic: cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent), asDB: appserviceDB, rsAPI: rsAPI, serverName: string(cfg.Global.ServerName), diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 75184d3b7..4550343c7 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -55,7 +55,7 @@ func AddPublicRoutes( syncProducer := &producers.SyncAPIProducer{ JetStream: js, - Topic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputClientData), + Topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputClientData), } routing.Setup( diff --git a/eduserver/eduserver.go b/eduserver/eduserver.go index 9fe8704cf..91208a400 100644 --- a/eduserver/eduserver.go +++ b/eduserver/eduserver.go @@ -48,9 +48,9 @@ func NewInternalAPI( Cache: eduCache, UserAPI: userAPI, JetStream: js, - OutputTypingEventTopic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputTypingEvent), - OutputSendToDeviceEventTopic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputSendToDeviceEvent), - OutputReceiptEventTopic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputReceiptEvent), + OutputTypingEventTopic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent), + OutputSendToDeviceEventTopic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), + OutputReceiptEventTopic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent), ServerName: cfg.Matrix.ServerName, } } diff --git a/federationapi/consumers/eduserver.go b/federationapi/consumers/eduserver.go index 1f81fa258..e14e60f47 100644 --- a/federationapi/consumers/eduserver.go +++ b/federationapi/consumers/eduserver.go @@ -58,9 +58,9 @@ func NewOutputEDUConsumer( db: store, ServerName: cfg.Matrix.ServerName, durable: cfg.Matrix.JetStream.Durable("FederationAPIEDUServerConsumer"), - typingTopic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputTypingEvent), - sendToDeviceTopic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputSendToDeviceEvent), - receiptTopic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputReceiptEvent), + typingTopic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent), + sendToDeviceTopic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), + receiptTopic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent), } } diff --git a/federationapi/consumers/keychange.go b/federationapi/consumers/keychange.go index 33d716d25..94e454359 100644 --- a/federationapi/consumers/keychange.go +++ b/federationapi/consumers/keychange.go @@ -55,8 +55,8 @@ func NewKeyChangeConsumer( return &KeyChangeConsumer{ ctx: process.Context(), jetstream: js, - durable: cfg.Matrix.JetStream.TopicFor("FederationAPIKeyChangeConsumer"), - topic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputKeyChangeEvent), + durable: cfg.Matrix.JetStream.Prefixed("FederationAPIKeyChangeConsumer"), + topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputKeyChangeEvent), queues: queues, db: store, serverName: cfg.Matrix.ServerName, diff --git a/federationapi/consumers/roomserver.go b/federationapi/consumers/roomserver.go index 989f7cf49..ff2c8e5d4 100644 --- a/federationapi/consumers/roomserver.go +++ b/federationapi/consumers/roomserver.go @@ -61,7 +61,7 @@ func NewOutputRoomEventConsumer( queues: queues, rsAPI: rsAPI, durable: cfg.Matrix.JetStream.Durable("FederationAPIRoomServerConsumer"), - topic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputRoomEvent), + topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputRoomEvent), } } diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go index cf66bd387..c557dfbaa 100644 --- a/keyserver/keyserver.go +++ b/keyserver/keyserver.go @@ -46,7 +46,7 @@ func NewInternalAPI( logrus.WithError(err).Panicf("failed to connect to key server database") } keyChangeProducer := &producers.KeyChange{ - Topic: string(cfg.Matrix.JetStream.TopicFor(jetstream.OutputKeyChangeEvent)), + Topic: string(cfg.Matrix.JetStream.Prefixed(jetstream.OutputKeyChangeEvent)), JetStream: js, DB: db, } diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go index 91001e418..f96cefcb3 100644 --- a/roomserver/internal/api.go +++ b/roomserver/internal/api.go @@ -90,6 +90,7 @@ func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.FederationInternalA r.KeyRing = keyRing r.Inputer = &input.Inputer{ + Cfg: r.Cfg, ProcessContext: r.ProcessContext, DB: r.DB, InputRoomEventTopic: r.InputRoomEventTopic, diff --git a/roomserver/internal/input/input.go b/roomserver/internal/input/input.go index c6e354611..6a8ae6d00 100644 --- a/roomserver/internal/input/input.go +++ b/roomserver/internal/input/input.go @@ -19,6 +19,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "sync" "time" @@ -29,6 +30,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/internal/query" "github.com/matrix-org/dendrite/roomserver/storage" + "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/gomatrixserverlib" @@ -45,7 +47,35 @@ var keyContentFields = map[string]string{ "m.room.member": "membership", } +// Inputer is responsible for consuming from the roomserver input +// streams and processing the events. All input events are queued +// into a single NATS stream and the order is preserved strictly. +// The `room_id` message header will contain the room ID which will +// be used to assign the pending event to a per-room worker. +// +// The input API maintains an ephemeral headers-only consumer. It +// will speed through the stream working out which room IDs are +// pending and create durable consumers for them. The durable +// consumer will then be used for each room worker goroutine to +// fetch events one by one and process them. Each room having a +// durable consumer of its own means there is no head-of-line +// blocking between rooms. Filtering ensures that each durable +// consumer only receives events for the room it is interested in. +// +// The ephemeral consumer closely tracks the newest events. The +// per-room durable consumers will only progress through the stream +// as events are processed. +// +// A BC * -> positions of each consumer (* = ephemeral) +// ⌄ ⌄⌄ ⌄ +// ABAABCAABCAA -> newest (letter = subject for each message) +// +// In this example, A is still processing an event but has two +// pending events to process afterwards. Both B and C are caught +// up, so they will do nothing until a new event comes in for B +// or C. type Inputer struct { + Cfg *config.RoomServer ProcessContext *process.ProcessContext DB storage.Database NATSClient *nats.Conn @@ -57,147 +87,275 @@ type Inputer struct { ACLs *acls.ServerACLs InputRoomEventTopic string OutputRoomEventTopic string - workers sync.Map // room ID -> *phony.Inbox + workers sync.Map // room ID -> *worker Queryer *query.Queryer } -func (r *Inputer) workerForRoom(roomID string) *phony.Inbox { - inbox, _ := r.workers.LoadOrStore(roomID, &phony.Inbox{}) - return inbox.(*phony.Inbox) +type worker struct { + phony.Inbox + sync.Mutex + r *Inputer + roomID string + subscription *nats.Subscription } -// eventsInProgress is an in-memory map to keep a track of which events we have -// queued up for processing. If we get a redelivery from NATS and we still have -// the queued up item then we won't do anything with the redelivered message. If -// we've restarted Dendrite and now this map is empty then it means that we will -// reload pending work from NATS. -var eventsInProgress sync.Map +func (r *Inputer) startWorkerForRoom(roomID string) { + v, loaded := r.workers.LoadOrStore(roomID, &worker{ + r: r, + roomID: roomID, + }) + w := v.(*worker) + w.Lock() + defer w.Unlock() + if !loaded || w.subscription == nil { + consumer := r.Cfg.Matrix.JetStream.Prefixed("RoomInput" + jetstream.Tokenise(w.roomID)) + subject := r.Cfg.Matrix.JetStream.Prefixed(jetstream.InputRoomEventSubj(w.roomID)) -// onMessage is called when a new event arrives in the roomserver input stream. + // Create the consumer. We do this as a specific step rather than + // letting PullSubscribe create it for us because we need the consumer + // to outlive the subscription. If we do it this way, we can Bind in the + // next step, and when we Unsubscribe, the consumer continues to live. If + // we leave PullSubscribe to create the durable consumer, Unsubscribe will + // delete it because it thinks it "owns" it, which in turn breaks the + // interest-based retention storage policy. + // If the durable consumer already exists, this is effectively a no-op. + // Another interesting tid-bit here: the ACK policy is set to "all" so that + // if we acknowledge a message, we also acknowledge everything that comes + // before it. This is necessary because otherwise our consumer will never + // acknowledge things we filtered out for other subjects and therefore they + // will linger around forever. + if _, err := w.r.JetStream.AddConsumer( + r.Cfg.Matrix.JetStream.Prefixed(jetstream.InputRoomEvent), + &nats.ConsumerConfig{ + Durable: consumer, + AckPolicy: nats.AckAllPolicy, + DeliverPolicy: nats.DeliverAllPolicy, + FilterSubject: subject, + AckWait: MaximumMissingProcessingTime + (time.Second * 10), + }, + ); err != nil { + logrus.WithError(err).Errorf("Failed to create consumer for room %q", w.roomID) + return + } + + // Bind to our durable consumer. We want to receive all messages waiting + // for this subject and we want to manually acknowledge them, so that we + // can ensure they are only cleaned up when we are done processing them. + sub, err := w.r.JetStream.PullSubscribe( + subject, consumer, + nats.ManualAck(), + nats.DeliverAll(), + nats.AckWait(MaximumMissingProcessingTime+(time.Second*10)), + nats.Bind(r.InputRoomEventTopic, consumer), + ) + if err != nil { + logrus.WithError(err).Errorf("Failed to subscribe to stream for room %q", w.roomID) + return + } + + // Go and start pulling messages off the queue. + w.subscription = sub + w.Act(nil, w._next) + } +} + +// Start creates an ephemeral non-durable consumer on the roomserver +// input topic. It is configured to deliver us headers only because we +// don't actually care about the contents of the message at this point, +// we only care about the `room_id` field. Once a message arrives, we +// will look to see if we have a worker for that room which has its +// own consumer. If we don't, we'll start one. func (r *Inputer) Start() error { _, err := r.JetStream.Subscribe( - r.InputRoomEventTopic, - // We specifically don't use jetstream.WithJetStreamMessage here because we - // queue the task off to a room-specific queue and the ACK needs to be sent - // later, possibly with an error response to the inputter if synchronous. - func(msg *nats.Msg) { - roomID := msg.Header.Get("room_id") - var inputRoomEvent api.InputRoomEvent - if err := json.Unmarshal(msg.Data, &inputRoomEvent); err != nil { - _ = msg.Term() - return - } - - _ = msg.InProgress() - index := roomID + "\000" + inputRoomEvent.Event.EventID() - if _, ok := eventsInProgress.LoadOrStore(index, struct{}{}); ok { - // We're already waiting to deal with this event, so there's no - // point in queuing it up again. We've notified NATS that we're - // working on the message still, so that will have deferred the - // redelivery by a bit. - return - } - - roomserverInputBackpressure.With(prometheus.Labels{"room_id": roomID}).Inc() - r.workerForRoom(roomID).Act(nil, func() { - _ = msg.InProgress() // resets the acknowledgement wait timer - defer eventsInProgress.Delete(index) - defer roomserverInputBackpressure.With(prometheus.Labels{"room_id": roomID}).Dec() - var errString string - if err := r.processRoomEvent(r.ProcessContext.Context(), &inputRoomEvent); err != nil { - if !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) { - sentry.CaptureException(err) - } - logrus.WithError(err).WithFields(logrus.Fields{ - "room_id": roomID, - "event_id": inputRoomEvent.Event.EventID(), - "type": inputRoomEvent.Event.Type(), - }).Warn("Roomserver failed to process async event") - _ = msg.Term() - errString = err.Error() - } else { - _ = msg.Ack() - } - if replyTo := msg.Header.Get("sync"); replyTo != "" { - if err := r.NATSClient.Publish(replyTo, []byte(errString)); err != nil { - logrus.WithError(err).WithFields(logrus.Fields{ - "room_id": roomID, - "event_id": inputRoomEvent.Event.EventID(), - "type": inputRoomEvent.Event.Type(), - }).Warn("Roomserver failed to respond for sync event") - } - } - }) + "", // This is blank because we specified it in BindStream. + func(m *nats.Msg) { + roomID := m.Header.Get(jetstream.RoomID) + r.startWorkerForRoom(roomID) + _ = m.Ack() }, - // NATS wants to acknowledge automatically by default when the message is - // read from the stream, but we want to override that behaviour by making - // sure that we only acknowledge when we're happy we've done everything we - // can. This ensures we retry things when it makes sense to do so. - nats.ManualAck(), - // Use a durable named consumer. - r.Durable, - // If we've missed things in the stream, e.g. we restarted, then replay - // all of the queued messages that were waiting for us. + nats.HeadersOnly(), nats.DeliverAll(), - // Ensure that NATS doesn't try to resend us something that wasn't done - // within the period of time that we might still be processing it. - nats.AckWait(MaximumMissingProcessingTime+(time.Second*10)), - // It is recommended to disable this for pull consumers as per the docs: - // https://docs.nats.io/nats-concepts/jetstream/consumers#note-about-push-and-pull-consumers - nats.MaxAckPending(-1), + nats.AckAll(), + nats.BindStream(r.InputRoomEventTopic), ) return err } +// _next is called by the worker for the room. It must only be called +// by the actor embedded into the worker. +func (w *worker) _next() { + // Look up what the next event is that's waiting to be processed. + ctx, cancel := context.WithTimeout(w.r.ProcessContext.Context(), time.Minute) + defer cancel() + msgs, err := w.subscription.Fetch(1, nats.Context(ctx)) + switch err { + case nil: + // Make sure that once we're done here, we queue up another call + // to _next in the inbox. + defer w.Act(nil, w._next) + + // If no error was reported, but we didn't get exactly one message, + // then skip over this and try again on the next iteration. + if len(msgs) != 1 { + return + } + + case context.DeadlineExceeded: + // The context exceeded, so we've been waiting for more than a + // minute for activity in this room. At this point we will shut + // down the subscriber to free up resources. It'll get started + // again if new activity happens. + if err = w.subscription.Unsubscribe(); err != nil { + logrus.WithError(err).Errorf("Failed to unsubscribe to stream for room %q", w.roomID) + } + w.Lock() + w.subscription = nil + w.Unlock() + return + + default: + // Something went wrong while trying to fetch the next event + // from the queue. In which case, we'll shut down the subscriber + // and wait to be notified about new room activity again. Maybe + // the problem will be corrected by then. + logrus.WithError(err).Errorf("Failed to get next stream message for room %q", w.roomID) + if err = w.subscription.Unsubscribe(); err != nil { + logrus.WithError(err).Errorf("Failed to unsubscribe to stream for room %q", w.roomID) + } + w.Lock() + w.subscription = nil + w.Unlock() + return + } + + // Try to unmarshal the input room event. If the JSON unmarshalling + // fails then we'll terminate the message — this notifies NATS that + // we are done with the message and never want to see it again. + msg := msgs[0] + var inputRoomEvent api.InputRoomEvent + if err = json.Unmarshal(msg.Data, &inputRoomEvent); err != nil { + _ = msg.Term() + return + } + + roomserverInputBackpressure.With(prometheus.Labels{"room_id": w.roomID}).Inc() + defer roomserverInputBackpressure.With(prometheus.Labels{"room_id": w.roomID}).Dec() + + // Process the room event. If something goes wrong then we'll tell + // NATS to terminate the message. We'll store the error result as + // a string, because we might want to return that to the caller if + // it was a synchronous request. + var errString string + if err = w.r.processRoomEvent(w.r.ProcessContext.Context(), &inputRoomEvent); err != nil { + if !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) { + sentry.CaptureException(err) + } + logrus.WithError(err).WithFields(logrus.Fields{ + "room_id": w.roomID, + "event_id": inputRoomEvent.Event.EventID(), + "type": inputRoomEvent.Event.Type(), + }).Warn("Roomserver failed to process async event") + _ = msg.Term() + errString = err.Error() + } else { + _ = msg.Ack() + } + + // If it was a synchronous input request then the "sync" field + // will be present in the message. That means that someone is + // waiting for a response. The temporary inbox name is present in + // that field, so send back the error string (if any). If there + // was no error then we'll return a blank message, which means + // that everything was OK. + if replyTo := msg.Header.Get("sync"); replyTo != "" { + if err = w.r.NATSClient.Publish(replyTo, []byte(errString)); err != nil { + logrus.WithError(err).WithFields(logrus.Fields{ + "room_id": w.roomID, + "event_id": inputRoomEvent.Event.EventID(), + "type": inputRoomEvent.Event.Type(), + }).Warn("Roomserver failed to respond for sync event") + } + } +} + +// queueInputRoomEvents queues events into the roomserver input +// stream in NATS. +func (r *Inputer) queueInputRoomEvents( + ctx context.Context, + request *api.InputRoomEventsRequest, +) (replySub *nats.Subscription, err error) { + // If the request is synchronous then we need to create a + // temporary inbox to wait for responses on, and then create + // a subscription to it. If it's asynchronous then we won't + // bother, so these values will remain empty. + var replyTo string + if !request.Asynchronous { + replyTo = nats.NewInbox() + replySub, err = r.NATSClient.SubscribeSync(replyTo) + if err != nil { + return nil, fmt.Errorf("r.NATSClient.SubscribeSync: %w", err) + } + if replySub == nil { + // This shouldn't ever happen, but it doesn't hurt to check + // because we can potentially avoid a nil pointer panic later + // if it did for some reason. + return nil, fmt.Errorf("expected a subscription to the temporary inbox") + } + } + + // For each event, marshal the input room event and then + // send it into the input queue. + for _, e := range request.InputRoomEvents { + roomID := e.Event.RoomID() + subj := r.Cfg.Matrix.JetStream.Prefixed(jetstream.InputRoomEventSubj(roomID)) + msg := &nats.Msg{ + Subject: subj, + Header: nats.Header{}, + } + msg.Header.Set("room_id", roomID) + if replyTo != "" { + msg.Header.Set("sync", replyTo) + } + msg.Data, err = json.Marshal(e) + if err != nil { + return nil, fmt.Errorf("json.Marshal: %w", err) + } + if _, err = r.JetStream.PublishMsg(msg, nats.Context(ctx)); err != nil { + logrus.WithError(err).WithFields(logrus.Fields{ + "room_id": roomID, + "event_id": e.Event.EventID(), + "subj": subj, + }).Error("Roomserver failed to queue async event") + return nil, fmt.Errorf("r.JetStream.PublishMsg: %w", err) + } + } + return +} + // InputRoomEvents implements api.RoomserverInternalAPI func (r *Inputer) InputRoomEvents( ctx context.Context, request *api.InputRoomEventsRequest, response *api.InputRoomEventsResponse, ) { - var replyTo string - var replySub *nats.Subscription - if !request.Asynchronous { - var err error - replyTo = nats.NewInbox() - replySub, err = r.NATSClient.SubscribeSync(replyTo) - if err != nil { - response.ErrMsg = err.Error() - return - } - } - - var err error - for _, e := range request.InputRoomEvents { - msg := &nats.Msg{ - Subject: r.InputRoomEventTopic, - Header: nats.Header{}, - Reply: replyTo, - } - roomID := e.Event.RoomID() - msg.Header.Set("room_id", roomID) - if replyTo != "" { - msg.Header.Set("sync", replyTo) - } - msg.Data, err = json.Marshal(e) - if err != nil { - response.ErrMsg = err.Error() - return - } - if _, err = r.JetStream.PublishMsg(msg); err != nil { - logrus.WithError(err).WithFields(logrus.Fields{ - "room_id": roomID, - "event_id": e.Event.EventID(), - }).Error("Roomserver failed to queue async event") - return - } - } - - if request.Asynchronous || replySub == nil { + // Queue up the event into the roomserver. + replySub, err := r.queueInputRoomEvents(ctx, request) + if err != nil { + response.ErrMsg = err.Error() return } + // If we aren't waiting for synchronous responses then we can + // give up here, there is nothing further to do. + if replySub == nil { + return + } + + // Otherwise, we'll want to sit and wait for the responses + // from the roomserver. There will be one response for every + // input we submitted. The last error value we receive will + // be the one returned as the error string. defer replySub.Drain() // nolint:errcheck for i := 0; i < len(request.InputRoomEvents); i++ { msg, err := replySub.NextMsgWithContext(ctx) @@ -207,7 +365,6 @@ func (r *Inputer) InputRoomEvents( } if len(msg.Data) > 0 { response.ErrMsg = string(msg.Data) - return } } } diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index 896773bab..36e3c5269 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -54,8 +54,8 @@ func NewInternalAPI( return internal.NewRoomserverAPI( base.ProcessContext, cfg, roomserverDB, js, nc, - cfg.Matrix.JetStream.TopicFor(jetstream.InputRoomEvent), - cfg.Matrix.JetStream.TopicFor(jetstream.OutputRoomEvent), + cfg.Matrix.JetStream.Prefixed(jetstream.InputRoomEvent), + cfg.Matrix.JetStream.Prefixed(jetstream.OutputRoomEvent), base.Caches, perspectiveServerNames, ) } diff --git a/setup/config/config_jetstream.go b/setup/config/config_jetstream.go index 9271cd8b4..b6a93d398 100644 --- a/setup/config/config_jetstream.go +++ b/setup/config/config_jetstream.go @@ -19,12 +19,12 @@ type JetStream struct { InMemory bool `yaml:"in_memory"` } -func (c *JetStream) TopicFor(name string) string { +func (c *JetStream) Prefixed(name string) string { return fmt.Sprintf("%s%s", c.TopicPrefix, name) } func (c *JetStream) Durable(name string) string { - return c.TopicFor(name) + return c.Prefixed(name) } func (c *JetStream) Defaults(generate bool) { diff --git a/setup/jetstream/nats.go b/setup/jetstream/nats.go index 43cc0331d..748c191b0 100644 --- a/setup/jetstream/nats.go +++ b/setup/jetstream/nats.go @@ -1,6 +1,7 @@ package jetstream import ( + "reflect" "strings" "sync" "time" @@ -75,14 +76,35 @@ func setupNATS(cfg *config.JetStream, nc *natsclient.Conn) (natsclient.JetStream } for _, stream := range streams { // streams are defined in streams.go - name := cfg.TopicFor(stream.Name) + name := cfg.Prefixed(stream.Name) info, err := s.StreamInfo(name) if err != nil && err != natsclient.ErrStreamNotFound { logrus.WithError(err).Fatal("Unable to get stream info") } + subjects := stream.Subjects + if len(subjects) == 0 { + // By default we want each stream to listen for the subjects + // that are either an exact match for the stream name, or where + // the first part of the subject is the stream name. ">" is a + // wildcard in NATS for one or more subject tokens. In the case + // that the stream is called "Foo", this will match any message + // with the subject "Foo", "Foo.Bar" or "Foo.Bar.Baz" etc. + subjects = []string{name, name + ".>"} + } + if info != nil { + switch { + case !reflect.DeepEqual(info.Config.Subjects, subjects): + fallthrough + case info.Config.Retention != stream.Retention: + fallthrough + case info.Config.Storage != stream.Storage: + if err = s.DeleteStream(name); err != nil { + logrus.WithError(err).Fatal("Unable to delete stream") + } + info = nil + } + } if info == nil { - stream.Subjects = []string{name} - // If we're trying to keep everything in memory (e.g. unit tests) // then overwrite the storage policy. if cfg.InMemory { @@ -93,8 +115,9 @@ func setupNATS(cfg *config.JetStream, nc *natsclient.Conn) (natsclient.JetStream // array, otherwise we end up with namespaces on namespaces. namespaced := *stream namespaced.Name = name + namespaced.Subjects = subjects if _, err = s.AddStream(&namespaced); err != nil { - logrus.WithError(err).WithField("stream", name).Fatal("Unable to add stream") + logrus.WithError(err).WithField("stream", name).WithField("subjects", subjects).Fatal("Unable to add stream") } } } diff --git a/setup/jetstream/streams.go b/setup/jetstream/streams.go index aa3e95cb8..aa979924b 100644 --- a/setup/jetstream/streams.go +++ b/setup/jetstream/streams.go @@ -1,6 +1,8 @@ package jetstream import ( + "fmt" + "regexp" "time" "github.com/nats-io/nats.go" @@ -24,10 +26,20 @@ var ( OutputReadUpdate = "OutputReadUpdate" ) +var safeCharacters = regexp.MustCompile("[^A-Za-z0-9$]+") + +func Tokenise(str string) string { + return safeCharacters.ReplaceAllString(str, "_") +} + +func InputRoomEventSubj(roomID string) string { + return fmt.Sprintf("%s.%s", InputRoomEvent, Tokenise(roomID)) +} + var streams = []*nats.StreamConfig{ { Name: InputRoomEvent, - Retention: nats.WorkQueuePolicy, + Retention: nats.InterestPolicy, Storage: nats.FileStorage, }, { diff --git a/syncapi/consumers/clientapi.go b/syncapi/consumers/clientapi.go index fcb7b5b1c..40c1cd3d6 100644 --- a/syncapi/consumers/clientapi.go +++ b/syncapi/consumers/clientapi.go @@ -61,7 +61,7 @@ func NewOutputClientDataConsumer( return &OutputClientDataConsumer{ ctx: process.Context(), jetstream: js, - topic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputClientData), + topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputClientData), durable: cfg.Matrix.JetStream.Durable("SyncAPIClientAPIConsumer"), db: store, notifier: notifier, diff --git a/syncapi/consumers/eduserver_receipts.go b/syncapi/consumers/eduserver_receipts.go index 4e4c61c67..ab79998ea 100644 --- a/syncapi/consumers/eduserver_receipts.go +++ b/syncapi/consumers/eduserver_receipts.go @@ -62,7 +62,7 @@ func NewOutputReceiptEventConsumer( return &OutputReceiptEventConsumer{ ctx: process.Context(), jetstream: js, - topic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputReceiptEvent), + topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent), durable: cfg.Matrix.JetStream.Durable("SyncAPIEDUServerReceiptConsumer"), db: store, notifier: notifier, diff --git a/syncapi/consumers/eduserver_sendtodevice.go b/syncapi/consumers/eduserver_sendtodevice.go index b0beef063..bdbe77352 100644 --- a/syncapi/consumers/eduserver_sendtodevice.go +++ b/syncapi/consumers/eduserver_sendtodevice.go @@ -57,7 +57,7 @@ func NewOutputSendToDeviceEventConsumer( return &OutputSendToDeviceEventConsumer{ ctx: process.Context(), jetstream: js, - topic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputSendToDeviceEvent), + topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), durable: cfg.Matrix.JetStream.Durable("SyncAPIEDUServerSendToDeviceConsumer"), db: store, serverName: cfg.Matrix.ServerName, diff --git a/syncapi/consumers/eduserver_typing.go b/syncapi/consumers/eduserver_typing.go index cae5df8a8..c2828c7fc 100644 --- a/syncapi/consumers/eduserver_typing.go +++ b/syncapi/consumers/eduserver_typing.go @@ -56,7 +56,7 @@ func NewOutputTypingEventConsumer( return &OutputTypingEventConsumer{ ctx: process.Context(), jetstream: js, - topic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputTypingEvent), + topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent), durable: cfg.Matrix.JetStream.Durable("SyncAPIEDUServerTypingConsumer"), eduCache: eduCache, notifier: notifier, diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 640c505c2..5bdc0fad7 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -65,7 +65,7 @@ func NewOutputRoomEventConsumer( ctx: process.Context(), cfg: cfg, jetstream: js, - topic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputRoomEvent), + topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputRoomEvent), durable: cfg.Matrix.JetStream.Durable("SyncAPIRoomServerConsumer"), db: store, notifier: notifier, diff --git a/syncapi/consumers/userapi.go b/syncapi/consumers/userapi.go index a3b2dd53d..010fa7c8e 100644 --- a/syncapi/consumers/userapi.go +++ b/syncapi/consumers/userapi.go @@ -56,7 +56,7 @@ func NewOutputNotificationDataConsumer( ctx: process.Context(), jetstream: js, durable: cfg.Matrix.JetStream.Durable("SyncAPINotificationDataConsumer"), - topic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputNotificationData), + topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputNotificationData), db: store, notifier: notifier, stream: stream, diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index f1f827221..ed8118bfc 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -67,18 +67,18 @@ func AddPublicRoutes( userAPIStreamEventProducer := &producers.UserAPIStreamEventProducer{ JetStream: js, - Topic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputStreamEvent), + Topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputStreamEvent), } userAPIReadUpdateProducer := &producers.UserAPIReadProducer{ JetStream: js, - Topic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputReadUpdate), + Topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputReadUpdate), } _ = userAPIReadUpdateProducer keyChangeConsumer := consumers.NewOutputKeyChangeEventConsumer( - process, cfg, cfg.Matrix.JetStream.TopicFor(jetstream.OutputKeyChangeEvent), + process, cfg, cfg.Matrix.JetStream.Prefixed(jetstream.OutputKeyChangeEvent), js, keyAPI, rsAPI, syncDB, notifier, streams.DeviceListStreamProvider, ) diff --git a/userapi/consumers/syncapi_readupdate.go b/userapi/consumers/syncapi_readupdate.go index 2e58020b4..067f93330 100644 --- a/userapi/consumers/syncapi_readupdate.go +++ b/userapi/consumers/syncapi_readupdate.go @@ -47,7 +47,7 @@ func NewOutputReadUpdateConsumer( db: store, ServerName: cfg.Matrix.ServerName, durable: cfg.Matrix.JetStream.Durable("UserAPISyncAPIReadUpdateConsumer"), - topic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputReadUpdate), + topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputReadUpdate), pgClient: pgClient, userAPI: userAPI, syncProducer: syncProducer, diff --git a/userapi/consumers/syncapi_streamevent.go b/userapi/consumers/syncapi_streamevent.go index 110813274..da3cd3937 100644 --- a/userapi/consumers/syncapi_streamevent.go +++ b/userapi/consumers/syncapi_streamevent.go @@ -54,7 +54,7 @@ func NewOutputStreamEventConsumer( jetstream: js, db: store, durable: cfg.Matrix.JetStream.Durable("UserAPISyncAPIStreamEventConsumer"), - topic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputStreamEvent), + topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputStreamEvent), pgClient: pgClient, userAPI: userAPI, rsAPI: rsAPI, diff --git a/userapi/userapi.go b/userapi/userapi.go index 97bdf7b2d..e91ce3a7a 100644 --- a/userapi/userapi.go +++ b/userapi/userapi.go @@ -54,8 +54,8 @@ func NewInternalAPI( // it's handled by clientapi, and hence uses its topic. When user // API handles it for all account data, we can remove it from // here. - cfg.Matrix.JetStream.TopicFor(jetstream.OutputClientData), - cfg.Matrix.JetStream.TopicFor(jetstream.OutputNotificationData), + cfg.Matrix.JetStream.Prefixed(jetstream.OutputClientData), + cfg.Matrix.JetStream.Prefixed(jetstream.OutputNotificationData), ) userAPI := &internal.UserInternalAPI{ From 3213910977db46dea7398ddb75d67584a99f57b4 Mon Sep 17 00:00:00 2001 From: kegsay Date: Wed, 23 Mar 2022 11:41:48 +0000 Subject: [PATCH 02/55] Update PR template (#2294) --- .github/PULL_REQUEST_TEMPLATE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 8014e9414..9bfb01667 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,6 +2,7 @@ +* [ ] I have added added tests for PR _or_ I have justified why this PR doesn't need tests. * [ ] Pull request includes a [sign off](https://github.com/matrix-org/dendrite/blob/main/docs/CONTRIBUTING.md#sign-off) Signed-off-by: `Your Name ` From 6e8cca344aa5ea4356d33e1594c725a6e60b5d5e Mon Sep 17 00:00:00 2001 From: S7evinK <2353100+S7evinK@users.noreply.github.com> Date: Wed, 23 Mar 2022 13:55:34 +0100 Subject: [PATCH 03/55] Fix #2287 by trying to fetch account by lowercase localpart (#2292) * Fix #2287 by trying to fetch account by lowercase localpart and as passed by request --- userapi/storage/shared/storage.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/userapi/storage/shared/storage.go b/userapi/storage/shared/storage.go index febf03221..72ae96ecc 100644 --- a/userapi/storage/shared/storage.go +++ b/userapi/storage/shared/storage.go @@ -23,6 +23,7 @@ import ( "errors" "fmt" "strconv" + "strings" "time" "github.com/matrix-org/gomatrixserverlib" @@ -298,7 +299,12 @@ func (d *Database) CheckAccountAvailability(ctx context.Context, localpart strin // Returns sql.ErrNoRows if no account exists which matches the given localpart. func (d *Database) GetAccountByLocalpart(ctx context.Context, localpart string, ) (*api.Account, error) { - return d.Accounts.SelectAccountByLocalpart(ctx, localpart) + // try to get the account with lowercase localpart (majority) + acc, err := d.Accounts.SelectAccountByLocalpart(ctx, strings.ToLower(localpart)) + if err == sql.ErrNoRows { + acc, err = d.Accounts.SelectAccountByLocalpart(ctx, localpart) // try with localpart as passed by the request + } + return acc, err } // SearchProfiles returns all profiles where the provided localpart or display name From 6c95f9b535191bc28cb983a6fa0eab036dc40916 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 23 Mar 2022 14:45:22 +0000 Subject: [PATCH 04/55] Update to matrix-org/pinecone@6fb0773 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a31030477..ae925c719 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 github.com/matrix-org/gomatrixserverlib v0.0.0-20220317164600-0980b7f341e0 - github.com/matrix-org/pinecone v0.0.0-20220318153125-8687b0d7072a + github.com/matrix-org/pinecone v0.0.0-20220323142759-6fb077377278 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.10 github.com/morikuni/aec v1.0.0 // indirect diff --git a/go.sum b/go.sum index 591ee1e53..25174f468 100644 --- a/go.sum +++ b/go.sum @@ -943,8 +943,8 @@ github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= github.com/matrix-org/gomatrixserverlib v0.0.0-20220317164600-0980b7f341e0 h1:IINbE/0jSYGb7M31StazufyIQdYWSivRlhuns3JYPOM= github.com/matrix-org/gomatrixserverlib v0.0.0-20220317164600-0980b7f341e0/go.mod h1:+WF5InseAMgi1fTnU46JH39IDpEvLep0fDzx9LDf2Bo= -github.com/matrix-org/pinecone v0.0.0-20220318153125-8687b0d7072a h1:4AKrIX2mZ9s7x3tCy17rvd+7ofWhWEMiI5xR09E6Ni4= -github.com/matrix-org/pinecone v0.0.0-20220318153125-8687b0d7072a/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk= +github.com/matrix-org/pinecone v0.0.0-20220323142759-6fb077377278 h1:lRrvMMv7x1FIVW1mcBdU89lvbgAXKz6RyYR0VQTAr3E= +github.com/matrix-org/pinecone v0.0.0-20220323142759-6fb077377278/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From d983d17355584b7de086389e069a935f1a5510a1 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 24 Mar 2022 10:03:22 +0000 Subject: [PATCH 05/55] Fix lint errors --- clientapi/auth/password.go | 2 +- keyserver/internal/internal.go | 1 + roomserver/api/alias.go | 1 - roomserver/api/alias_test.go | 16 ++++----- roomserver/internal/alias.go | 9 +++-- .../storage/sqlite3/event_state_keys_table.go | 2 +- .../storage/sqlite3/event_types_table.go | 2 +- roomserver/storage/sqlite3/events_table.go | 4 +-- setup/mscs/msc2946/msc2946.go | 33 ++++--------------- .../postgres/output_room_events_table.go | 4 +-- .../sqlite3/output_room_events_table.go | 4 +-- 11 files changed, 30 insertions(+), 48 deletions(-) diff --git a/clientapi/auth/password.go b/clientapi/auth/password.go index 18cf94979..046b36f0b 100644 --- a/clientapi/auth/password.go +++ b/clientapi/auth/password.go @@ -62,7 +62,7 @@ func (t *LoginTypePassword) LoginFromJSON(ctx context.Context, reqBytes []byte) func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, *util.JSONResponse) { r := req.(*PasswordRequest) - username := strings.ToLower(r.Username()) + username := strings.ToLower(r.Username()) if username == "" { return nil, &util.JSONResponse{ Code: http.StatusUnauthorized, diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index cc9d3a616..a05476f5f 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -223,6 +223,7 @@ func (a *KeyInternalAPI) QueryDeviceMessages(ctx context.Context, req *api.Query res.StreamID = maxStreamID } +// nolint:gocyclo func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysRequest, res *api.QueryKeysResponse) { res.DeviceKeys = make(map[string]map[string]json.RawMessage) res.MasterKeys = make(map[string]gomatrixserverlib.CrossSigningKey) diff --git a/roomserver/api/alias.go b/roomserver/api/alias.go index be37333b6..baab27751 100644 --- a/roomserver/api/alias.go +++ b/roomserver/api/alias.go @@ -102,4 +102,3 @@ func (a AliasEvent) Valid() bool { } return a.Alias == "" || validateAliasRegex.MatchString(a.Alias) } - diff --git a/roomserver/api/alias_test.go b/roomserver/api/alias_test.go index 680493b7b..686f064bb 100644 --- a/roomserver/api/alias_test.go +++ b/roomserver/api/alias_test.go @@ -22,29 +22,29 @@ func TestAliasEvent_Valid(t *testing.T) { { name: "empty alias, invalid alt aliases", fields: fields{ - Alias: "", - AltAliases: []string{ "%not:valid.local"}, + Alias: "", + AltAliases: []string{"%not:valid.local"}, }, }, { name: "valid alias, invalid alt aliases", fields: fields{ - Alias: "#valid:test.local", - AltAliases: []string{ "%not:valid.local"}, + Alias: "#valid:test.local", + AltAliases: []string{"%not:valid.local"}, }, }, { name: "empty alias, invalid alt aliases", fields: fields{ - Alias: "", - AltAliases: []string{ "%not:valid.local"}, + Alias: "", + AltAliases: []string{"%not:valid.local"}, }, }, { name: "invalid alias", fields: fields{ - Alias: "%not:valid.local", - AltAliases: []string{ }, + Alias: "%not:valid.local", + AltAliases: []string{}, }, }, } diff --git a/roomserver/internal/alias.go b/roomserver/internal/alias.go index 5c1c04f01..02fc4a5a7 100644 --- a/roomserver/internal/alias.go +++ b/roomserver/internal/alias.go @@ -173,12 +173,15 @@ func (r *RoomserverInternalAPI) RemoveRoomAlias( } if creatorID != request.UserID { - plEvent, err := r.DB.GetStateEvent(ctx, roomID, gomatrixserverlib.MRoomPowerLevels, "") + var plEvent *gomatrixserverlib.HeaderedEvent + var pls *gomatrixserverlib.PowerLevelContent + + plEvent, err = r.DB.GetStateEvent(ctx, roomID, gomatrixserverlib.MRoomPowerLevels, "") if err != nil { return fmt.Errorf("r.DB.GetStateEvent: %w", err) } - pls, err := plEvent.PowerLevels() + pls, err = plEvent.PowerLevels() if err != nil { return fmt.Errorf("plEvent.PowerLevels: %w", err) } @@ -223,7 +226,7 @@ func (r *RoomserverInternalAPI) RemoveRoomAlias( } stateRes := &api.QueryLatestEventsAndStateResponse{} - if err := helpers.QueryLatestEventsAndState(ctx, r.DB, &api.QueryLatestEventsAndStateRequest{RoomID: roomID, StateToFetch: eventsNeeded.Tuples()}, stateRes); err != nil { + if err = helpers.QueryLatestEventsAndState(ctx, r.DB, &api.QueryLatestEventsAndStateRequest{RoomID: roomID, StateToFetch: eventsNeeded.Tuples()}, stateRes); err != nil { return err } diff --git a/roomserver/storage/sqlite3/event_state_keys_table.go b/roomserver/storage/sqlite3/event_state_keys_table.go index 6ae3ab0c4..f97541f4a 100644 --- a/roomserver/storage/sqlite3/event_state_keys_table.go +++ b/roomserver/storage/sqlite3/event_state_keys_table.go @@ -151,7 +151,7 @@ func (s *eventStateKeyStatements) BulkSelectEventStateKey( if err != nil { return nil, err } - defer selectPrep.Close() + defer internal.CloseAndLogIfError(ctx, selectPrep, "selectPrep.close() failed") stmt := sqlutil.TxStmt(txn, selectPrep) rows, err := stmt.QueryContext(ctx, iEventStateKeyNIDs...) if err != nil { diff --git a/roomserver/storage/sqlite3/event_types_table.go b/roomserver/storage/sqlite3/event_types_table.go index 1fe4c91c2..c49cc509a 100644 --- a/roomserver/storage/sqlite3/event_types_table.go +++ b/roomserver/storage/sqlite3/event_types_table.go @@ -128,7 +128,7 @@ func (s *eventTypeStatements) BulkSelectEventTypeNID( if err != nil { return nil, err } - defer selectPrep.Close() + defer internal.CloseAndLogIfError(ctx, selectPrep, "selectPrep.close() failed") stmt := sqlutil.TxStmt(txn, selectPrep) /////////////// diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index 2ab1151d5..45b49e5cb 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -567,7 +567,7 @@ func (s *eventStatements) SelectMaxEventDepth(ctx context.Context, txn *sql.Tx, if err != nil { return 0, err } - defer sqlPrep.Close() + defer internal.CloseAndLogIfError(ctx, sqlPrep, "sqlPrep.close() failed") err = sqlutil.TxStmt(txn, sqlPrep).QueryRowContext(ctx, iEventIDs...).Scan(&result) if err != nil { return 0, fmt.Errorf("sqlutil.TxStmt.QueryRowContext: %w", err) @@ -583,7 +583,7 @@ func (s *eventStatements) SelectRoomNIDsForEventNIDs( if err != nil { return nil, err } - defer sqlPrep.Close() + defer internal.CloseAndLogIfError(ctx, sqlPrep, "sqlPrep.close() failed") sqlStmt := sqlutil.TxStmt(txn, sqlPrep) iEventNIDs := make([]interface{}, len(eventNIDs)) for i, v := range eventNIDs { diff --git a/setup/mscs/msc2946/msc2946.go b/setup/mscs/msc2946/msc2946.go index 7fb043366..61520d50e 100644 --- a/setup/mscs/msc2946/msc2946.go +++ b/setup/mscs/msc2946/msc2946.go @@ -283,11 +283,7 @@ func (w *walker) walk() util.JSONResponse { if !roomExists { // attempt to query this room over federation, as either we've never heard of it before // or we've left it and hence are not authorised (but info may be exposed regardless) - fedRes, err := w.federatedRoomInfo(rv.roomID, rv.vias) - if err != nil { - util.GetLogger(w.ctx).WithError(err).WithField("room_id", rv.roomID).Errorf("failed to query federated spaces") - continue - } + fedRes := w.federatedRoomInfo(rv.roomID, rv.vias) if fedRes != nil { discoveredChildEvents = fedRes.Room.ChildrenState discoveredRooms = append(discoveredRooms, fedRes.Room) @@ -420,15 +416,15 @@ func (w *walker) publicRoomsChunk(roomID string) *gomatrixserverlib.PublicRoom { // federatedRoomInfo returns more of the spaces graph from another server. Returns nil if this was // unsuccessful. -func (w *walker) federatedRoomInfo(roomID string, vias []string) (*gomatrixserverlib.MSC2946SpacesResponse, error) { +func (w *walker) federatedRoomInfo(roomID string, vias []string) *gomatrixserverlib.MSC2946SpacesResponse { // only do federated requests for client requests if w.caller == nil { - return nil, nil + return nil } resp, ok := w.cache.GetSpaceSummary(roomID) if ok { util.GetLogger(w.ctx).Debugf("Returning cached response for %s", roomID) - return &resp, nil + return &resp } util.GetLogger(w.ctx).Debugf("Querying %s via %+v", roomID, vias) ctx := context.Background() @@ -455,9 +451,9 @@ func (w *walker) federatedRoomInfo(roomID string, vias []string) (*gomatrixserve } w.cache.StoreSpaceSummary(roomID, res) - return &res, nil + return &res } - return nil, nil + return nil } func (w *walker) roomExists(roomID string) bool { @@ -717,23 +713,6 @@ func stripped(ev *gomatrixserverlib.Event) *gomatrixserverlib.MSC2946StrippedEve } } -func eventKey(event *gomatrixserverlib.MSC2946StrippedEvent) string { - return event.RoomID + "|" + event.Type + "|" + event.StateKey -} - -func spaceTargetStripped(event *gomatrixserverlib.MSC2946StrippedEvent) string { - if event.StateKey == "" { - return "" // no-op - } - switch event.Type { - case ConstSpaceParentEventType: - return event.StateKey - case ConstSpaceChildEventType: - return event.StateKey - } - return "" -} - func parseInt(intstr string, defaultVal int) int { i, err := strconv.ParseInt(intstr, 10, 32) if err != nil { diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index 26689f447..14af6a949 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -472,7 +472,7 @@ func (s *outputRoomEventsStatements) SelectContextBeforeEvent( if err != nil { return } - defer rows.Close() + defer internal.CloseAndLogIfError(ctx, rows, "rows.close() failed") for rows.Next() { var ( @@ -504,7 +504,7 @@ func (s *outputRoomEventsStatements) SelectContextAfterEvent( if err != nil { return } - defer rows.Close() + defer internal.CloseAndLogIfError(ctx, rows, "rows.close() failed") for rows.Next() { var ( diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index b9115262e..acd959696 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -514,7 +514,7 @@ func (s *outputRoomEventsStatements) SelectContextBeforeEvent( if err != nil { return } - defer rows.Close() + defer internal.CloseAndLogIfError(ctx, rows, "rows.close() failed") for rows.Next() { var ( @@ -550,7 +550,7 @@ func (s *outputRoomEventsStatements) SelectContextAfterEvent( if err != nil { return } - defer rows.Close() + defer internal.CloseAndLogIfError(ctx, rows, "rows.close() failed") for rows.Next() { var ( From 01f863d248467516ad9bb7d43239cf678dde4d7e Mon Sep 17 00:00:00 2001 From: S7evinK <2353100+S7evinK@users.noreply.github.com> Date: Thu, 24 Mar 2022 12:52:51 +0100 Subject: [PATCH 06/55] Move CI to Github Actions (#2297) * Initial test * Move CI to GHA * Naming * Always report all linter issues * Remove if true * Test complement in different variations * Try again * Move Complement back after initial tests and readd timeout Make linting fail further checks Remove CodeQL * Update and rename tests.yml to dendrite.yml Co-authored-by: Neil Alexander --- .github/workflows/codeql-analysis.yml | 34 --- .github/workflows/dendrite.yml | 317 ++++++++++++++++++++ .github/workflows/tests.yml | 71 ----- .github/workflows/wasm.yml | 49 --- build/scripts/Complement.Dockerfile | 3 +- build/scripts/ComplementPostgres.Dockerfile | 3 +- 6 files changed, 321 insertions(+), 156 deletions(-) delete mode 100644 .github/workflows/codeql-analysis.yml create mode 100644 .github/workflows/dendrite.yml delete mode 100644 .github/workflows/tests.yml delete mode 100644 .github/workflows/wasm.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index de6c79ddc..000000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: "CodeQL" - -on: - push: - branches: [main] - pull_request: - branches: [main] - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - language: ["go"] - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - with: - fetch-depth: 2 - - - run: git checkout HEAD^2 - if: ${{ github.event_name == 'pull_request' }} - - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: ${{ matrix.language }} - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml new file mode 100644 index 000000000..4fcffa0bb --- /dev/null +++ b/.github/workflows/dendrite.yml @@ -0,0 +1,317 @@ +name: Dendrite + +on: + push: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + wasm: + name: WASM build test + timeout-minutes: 5 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: 1.16 + + - uses: actions/cache@v2 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-wasm-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-wasm + + - name: Install Node + uses: actions/setup-node@v2 + with: + node-version: 14 + + - uses: actions/cache@v2 + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node- + + - name: Reconfigure Git to use HTTPS auth for repo packages + run: > + git config --global url."https://github.com/".insteadOf + ssh://git@github.com/ + + - name: Install test dependencies + working-directory: ./test/wasm + run: npm ci + + - name: Test + run: ./test-dendritejs.sh + + # Run golangci-lint + lint: + timeout-minutes: 5 + name: Linting + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: golangci-lint + uses: golangci/golangci-lint-action@v2 + + # run go test with different go versions + test: + timeout-minutes: 5 + name: Unit tests (Go ${{ matrix.go }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + go: [ '1.16', '1.17', '1.18' ] + steps: + - uses: actions/checkout@v3 + - name: Setup go + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go }} + - uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go${{ matrix.go }}-test-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go${{ matrix.go }}-test- + - run: go test ./... + + # build Dendrite for linux with different architectures and go versions + build: + name: Build for Linux + timeout-minutes: 10 + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + go: [ '1.16', '1.17', '1.18' ] + goos: [ 'linux' ] + goarch: [ 'amd64', '386' ] + steps: + - uses: actions/checkout@v3 + - name: Setup go + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go }} + - name: Install dependencies x86 + if: ${{ matrix.goarch == '386' }} + run: sudo apt update && sudo apt-get install -y gcc-multilib + - uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goarch }}- + - env: + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + CGO_ENABLED: 1 + run: go build -trimpath -v -o "bin/" ./cmd/... + + # build for Windows 64-bit + build_windows: + name: Build for Windows + timeout-minutes: 10 + runs-on: ubuntu-latest + strategy: + matrix: + go: [ '1.16', '1.17', '1.18' ] + goos: [ 'windows' ] + goarch: [ 'amd64' ] + steps: + - uses: actions/checkout@v3 + - name: Setup Go ${{ matrix.go }} + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go }} + - name: Install dependencies + run: sudo apt update && sudo apt install -y gcc-mingw-w64-x86-64 # install required gcc + - uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goos }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goos }} + - env: + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + CGO_ENABLED: 1 + CC: "/usr/bin/x86_64-w64-mingw32-gcc" + run: go build -trimpath -v -o "bin/" ./cmd/... + + # Dummy step to gate other tests on without repeating the whole list + initial-tests-done: + name: Initial tests passed + if: ${{ !cancelled() }} # Run this even if prior jobs were skipped + needs: [ lint, test, build, build_windows ] + runs-on: ubuntu-latest + steps: + - run: "true" + + # run database upgrade tests + upgrade_test: + name: Upgrade tests + timeout-minutes: 20 + needs: initial-tests-done + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Setup go + uses: actions/setup-go@v2 + with: + go-version: '1.16' + - uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-upgrade-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-upgrade + - name: Build upgrade-tests + run: go build ./cmd/dendrite-upgrade-tests + - name: Test upgrade + run: ./dendrite-upgrade-tests --head . + + # run Sytest in different variations + sytest: + timeout-minutes: 20 + needs: initial-tests-done + name: "Sytest (${{ matrix.label }})" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - label: SQLite + + - label: SQLite, full HTTP APIs + api: full-http + + - label: PostgreSQL + postgres: postgres + + - label: PostgreSQL, full HTTP APIs + postgres: postgres + api: full-http + container: + image: matrixdotorg/sytest-dendrite:latest + volumes: + - ${{ github.workspace }}:/src + env: + POSTGRES: ${{ matrix.postgres && 1}} + API: ${{ matrix.api && 1 }} + steps: + - uses: actions/checkout@v2 + - name: Run Sytest + run: /bootstrap.sh dendrite + working-directory: /src + - name: Summarise results.tap + if: ${{ always() }} + run: /sytest/scripts/tap_to_gha.pl /logs/results.tap + + - name: Upload Sytest logs + uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }}) + path: | + /logs/results.tap + /logs/**/*.log* + + # run Complement + complement: + name: "Complement (${{ matrix.label }})" + timeout-minutes: 20 + needs: initial-tests-done + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - label: SQLite + + - label: SQLite, full HTTP APIs + api: full-http + + - label: PostgreSQL + postgres: Postgres + + - label: PostgreSQL, full HTTP APIs + postgres: Postgres + api: full-http + steps: + # Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env to run Complement. + # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path + - name: "Set Go Version" + run: | + echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH + echo "~/go/bin" >> $GITHUB_PATH + + - name: "Install Complement Dependencies" + # We don't need to install Go because it is included on the Ubuntu 20.04 image: + # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64 + run: | + sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev + go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest + + - name: Run actions/checkout@v2 for dendrite + uses: actions/checkout@v2 + with: + path: dendrite + + # Attempt to check out the same branch of Complement as the PR. If it + # doesn't exist, fallback to main. + - name: Checkout complement + shell: bash + run: | + mkdir -p complement + # Attempt to use the version of complement which best matches the current + # build. Depending on whether this is a PR or release, etc. we need to + # use different fallbacks. + # + # 1. First check if there's a similarly named branch (GITHUB_HEAD_REF + # for pull requests, otherwise GITHUB_REF). + # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y + # (GITHUB_BASE_REF for pull requests). + # 3. Use the default complement branch ("master"). + for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do + # Skip empty branch names and merge commits. + if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then + continue + fi + + (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break + done + + # Build initial Dendrite image + - run: docker build -t complement-dendrite -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile . + working-directory: dendrite + + # Run Complement + - run: | + set -o pipefail && + go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt + shell: bash + name: Run Complement Tests + env: + COMPLEMENT_BASE_IMAGE: complement-dendrite:latest + API: ${{ matrix.api && 1 }} + working-directory: complement diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index 4a1720295..000000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: Tests - -on: - push: - branches: ["main"] - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - complement: - runs-on: ubuntu-latest - steps: - # Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env to run Complement. - # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path - - name: "Set Go Version" - run: | - echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH - echo "~/go/bin" >> $GITHUB_PATH - - - name: "Install Complement Dependencies" - # We don't need to install Go because it is included on the Ubuntu 20.04 image: - # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64 - run: | - sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev - go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest - - - name: Run actions/checkout@v2 for dendrite - uses: actions/checkout@v2 - with: - path: dendrite - - # Attempt to check out the same branch of Complement as the PR. If it - # doesn't exist, fallback to main. - - name: Checkout complement - shell: bash - run: | - mkdir -p complement - # Attempt to use the version of complement which best matches the current - # build. Depending on whether this is a PR or release, etc. we need to - # use different fallbacks. - # - # 1. First check if there's a similarly named branch (GITHUB_HEAD_REF - # for pull requests, otherwise GITHUB_REF). - # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y - # (GITHUB_BASE_REF for pull requests). - # 3. Use the default complement branch ("master"). - for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do - # Skip empty branch names and merge commits. - if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then - continue - fi - - (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break - done - - # Build initial Dendrite image - - run: docker build -t complement-dendrite -f build/scripts/Complement.Dockerfile . - working-directory: dendrite - - # Run Complement - - run: | - set -o pipefail && - go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt - shell: bash - name: Run Complement Tests - env: - COMPLEMENT_BASE_IMAGE: complement-dendrite:latest - working-directory: complement diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml deleted file mode 100644 index 4889283af..000000000 --- a/.github/workflows/wasm.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: WebAssembly - -on: - push: - pull_request: - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Install Go - uses: actions/setup-go@v2 - with: - go-version: 1.16.5 - - - uses: actions/cache@v2 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - - name: Install Node - uses: actions/setup-node@v2 - with: - node-version: 14 - - - uses: actions/cache@v2 - with: - path: ~/.npm - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-node- - - - name: Reconfigure Git to use HTTPS auth for repo packages - run: > - git config --global url."https://github.com/".insteadOf - ssh://git@github.com/ - - - name: Install test dependencies - working-directory: ./test/wasm - run: npm ci - - - name: Test - run: ./test-dendritejs.sh diff --git a/build/scripts/Complement.Dockerfile b/build/scripts/Complement.Dockerfile index 1d520b4e7..6b2942d97 100644 --- a/build/scripts/Complement.Dockerfile +++ b/build/scripts/Complement.Dockerfile @@ -21,6 +21,7 @@ WORKDIR /dendrite RUN ./generate-keys --private-key matrix_key.pem ENV SERVER_NAME=localhost +ENV API=0 EXPOSE 8008 8448 # At runtime, generate TLS cert based on the CA now mounted at /ca @@ -28,4 +29,4 @@ EXPOSE 8008 8448 CMD ./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \ ./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \ cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \ - ./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml + ./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0} diff --git a/build/scripts/ComplementPostgres.Dockerfile b/build/scripts/ComplementPostgres.Dockerfile index 6024ae8da..b98f4671c 100644 --- a/build/scripts/ComplementPostgres.Dockerfile +++ b/build/scripts/ComplementPostgres.Dockerfile @@ -39,6 +39,7 @@ WORKDIR /dendrite RUN ./generate-keys --private-key matrix_key.pem ENV SERVER_NAME=localhost +ENV API=0 EXPOSE 8008 8448 @@ -50,4 +51,4 @@ CMD /build/run_postgres.sh && ./generate-keys --server $SERVER_NAME --tls-cert s sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml && \ sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml && \ cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \ - ./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml \ No newline at end of file + ./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0} \ No newline at end of file From 3bdda65bc4cffe5c0c0b7db20e99c16afe929b8c Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 24 Mar 2022 12:05:36 +0000 Subject: [PATCH 07/55] Hopefully fail `Initial tests passed` if they actually failed (#2298) --- .github/workflows/dendrite.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index 4fcffa0bb..c1bd5a051 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -159,7 +159,6 @@ jobs: # Dummy step to gate other tests on without repeating the whole list initial-tests-done: name: Initial tests passed - if: ${{ !cancelled() }} # Run this even if prior jobs were skipped needs: [ lint, test, build, build_windows ] runs-on: ubuntu-latest steps: From fe5148c12da417ff9aba59a9ceb1e382d6b7507e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 24 Mar 2022 12:14:35 +0000 Subject: [PATCH 08/55] Check for success of initial tests on GHA --- .github/workflows/dendrite.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index c1bd5a051..79d8dddef 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -158,11 +158,15 @@ jobs: # Dummy step to gate other tests on without repeating the whole list initial-tests-done: - name: Initial tests passed - needs: [ lint, test, build, build_windows ] - runs-on: ubuntu-latest - steps: - - run: "true" + name: Initial tests passed + needs: [ lint, test, build, build_windows ] + runs-on: ubuntu-latest + if: ${{ !cancelled() }} # Run this even if prior jobs were skipped + steps: + - name: Check initial tests passed + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} # run database upgrade tests upgrade_test: From 41d71a304ca169bb192337fd1c6688ca07b16543 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 24 Mar 2022 12:25:45 +0000 Subject: [PATCH 09/55] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d3a862587..b4a9a614a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -# Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg?branch=master)](https://buildkite.com/matrix-dot-org/dendrite) [![Dendrite](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) [![Dendrite Dev](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) +# Dendrite +[![Build status](https://github.com/matrix-org/dendrite/actions/workflows/dendrite.yml/badge.svg?event=push)](https://github.com/matrix-org/dendrite/actions/workflows/dendrite.yml) [![Dendrite](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) [![Dendrite Dev](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) Dendrite is a second-generation Matrix homeserver written in Go. It intends to provide an **efficient**, **reliable** and **scalable** alternative to [Synapse](https://github.com/matrix-org/synapse): From c2a27efc365f0f68252561bd2a0404d9d64d8ff4 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 24 Mar 2022 12:40:44 +0000 Subject: [PATCH 10/55] Update `on` section of GHA workflow --- .github/workflows/dendrite.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index 79d8dddef..053b39474 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -2,7 +2,12 @@ name: Dendrite on: push: + branches: + - main pull_request: + release: + types: [published] + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} From ad818a43707feafaba89f726c925c184cc00eeb3 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 24 Mar 2022 12:41:30 +0000 Subject: [PATCH 11/55] Update dendrite.yml --- .github/workflows/dendrite.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index 053b39474..d337865f2 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -7,7 +7,6 @@ on: pull_request: release: types: [published] - workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} From bb31b25f1b3c7330a33b31f310aa73f99457031e Mon Sep 17 00:00:00 2001 From: Nick Cao Date: Thu, 24 Mar 2022 21:57:00 +0800 Subject: [PATCH 12/55] fixup treat the sender_localpart as an exclusive namespace of one user (#2255) Signed-off-by: Nick Cao Co-authored-by: Neil Alexander --- setup/config/config_appservice.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/setup/config/config_appservice.go b/setup/config/config_appservice.go index 4f6553f10..3f4e1c917 100644 --- a/setup/config/config_appservice.go +++ b/setup/config/config_appservice.go @@ -209,13 +209,14 @@ func setupRegexps(asAPI *AppServiceAPI, derived *Derived) (err error) { for _, appservice := range derived.ApplicationServices { // The sender_localpart can be considered an exclusive regex for a single user, so let's do that // to simplify the code - var senderUserIDSlice = []string{fmt.Sprintf("@%s:%s", appservice.SenderLocalpart, asAPI.Matrix.ServerName)} - usersSlice, found := appservice.NamespaceMap["users"] + users, found := appservice.NamespaceMap["users"] if !found { - usersSlice = []ApplicationServiceNamespace{} - appservice.NamespaceMap["users"] = usersSlice + users = []ApplicationServiceNamespace{} } - appendExclusiveNamespaceRegexs(&senderUserIDSlice, usersSlice) + appservice.NamespaceMap["users"] = append(users, ApplicationServiceNamespace{ + Exclusive: true, + Regex: regexp.QuoteMeta(fmt.Sprintf("@%s:%s", appservice.SenderLocalpart, asAPI.Matrix.ServerName)), + }) for key, namespaceSlice := range appservice.NamespaceMap { switch key { From 1b389abbfd26e949b2d511c64f20b5575aaecf79 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 24 Mar 2022 15:22:06 +0000 Subject: [PATCH 13/55] Upload Docker images for releases to both Docker Hub and GitHub Container Registry (#2299) * Upload Docker images for releases to both Docker Hub and GitHub Container Registry * Build current images on `:main` tag * Use Dendrite flow to trigger Docker flow for `:main` * Tweaks * Fix references to `env.GHCR_NAMESPACE` --- .github/workflows/docker-hub.yml | 71 ---------------- .github/workflows/docker.yml | 136 +++++++++++++++++++++++++++++++ build/docker/Dockerfile.monolith | 4 + build/docker/Dockerfile.polylith | 4 + 4 files changed, 144 insertions(+), 71 deletions(-) delete mode 100644 .github/workflows/docker-hub.yml create mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/docker-hub.yml b/.github/workflows/docker-hub.yml deleted file mode 100644 index 0322866d7..000000000 --- a/.github/workflows/docker-hub.yml +++ /dev/null @@ -1,71 +0,0 @@ -# Based on https://github.com/docker/build-push-action - -name: "Docker Hub" - -on: - release: - types: [published] - -env: - DOCKER_NAMESPACE: matrixdotorg - DOCKER_HUB_USER: dendritegithub - PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7 - -jobs: - Monolith: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Get release tag - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ env.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_TOKEN }} - - - name: Build monolith image - id: docker_build_monolith - uses: docker/build-push-action@v2 - with: - context: . - file: ./build/docker/Dockerfile.monolith - platforms: ${{ env.PLATFORMS }} - push: true - tags: | - ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:latest - ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ env.RELEASE_VERSION }} - - Polylith: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Get release tag - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ env.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_TOKEN }} - - - name: Build polylith image - id: docker_build_polylith - uses: docker/build-push-action@v2 - with: - context: . - file: ./build/docker/Dockerfile.polylith - platforms: ${{ env.PLATFORMS }} - push: true - tags: | - ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:latest - ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 000000000..129a9f0a1 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,136 @@ +# Based on https://github.com/docker/build-push-action + +name: "Docker Hub" + +on: + release: # A GitHub release was published + types: [published] + workflow_run: # The Dendrite pipeline completed successfully on main + workflows: [Dendrite] + types: [completed] + branches: [main] + +env: + DOCKER_NAMESPACE: matrixdotorg + DOCKER_HUB_USER: dendritegithub + GHCR_NAMESPACE: matrix-org + PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7 + +jobs: + monolith: + name: Monolith image + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Get release tag + if: github.event_name == 'release' # Only for GitHub releases + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ env.DOCKER_HUB_USER }} + password: ${{ secrets.DOCKER_TOKEN }} + - name: Login to GitHub Containers + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build monolith image + if: >- + github.event_name == 'workflow_run' && + github.event.workflow_run.conclusion == 'success' + id: docker_build_monolith + uses: docker/build-push-action@v2 + with: + cache-from: type=gha + cache-to: type=gha,mode=max + context: . + file: ./build/docker/Dockerfile.monolith + platforms: ${{ env.PLATFORMS }} + push: true + tags: | + ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:main + ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:main + + - name: Build release monolith image + if: github.event_name == 'release' # Only for GitHub releases + id: docker_build_monolith + uses: docker/build-push-action@v2 + with: + cache-from: type=gha + cache-to: type=gha,mode=max + context: . + file: ./build/docker/Dockerfile.monolith + platforms: ${{ env.PLATFORMS }} + push: true + tags: | + ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:latest + ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ env.RELEASE_VERSION }} + ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:latest + ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ env.RELEASE_VERSION }} + + polylith: + name: Polylith image + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Get release tag + if: github.event_name == 'release' # Only for GitHub releases + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ env.DOCKER_HUB_USER }} + password: ${{ secrets.DOCKER_TOKEN }} + - name: Login to GitHub Containers + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build polylith image + if: >- + github.event_name == 'workflow_run' && + github.event.workflow_run.conclusion == 'success' + id: docker_build_polylith + uses: docker/build-push-action@v2 + with: + cache-from: type=gha + cache-to: type=gha,mode=max + context: . + file: ./build/docker/Dockerfile.polylith + platforms: ${{ env.PLATFORMS }} + push: true + tags: | + ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:main + ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:main + + - name: Build release polylith image + if: github.event_name == 'release' # Only for GitHub releases + id: docker_build_polylith + uses: docker/build-push-action@v2 + with: + cache-from: type=gha + cache-to: type=gha,mode=max + context: . + file: ./build/docker/Dockerfile.polylith + platforms: ${{ env.PLATFORMS }} + push: true + tags: | + ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:latest + ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }} + ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:latest + ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }} diff --git a/build/docker/Dockerfile.monolith b/build/docker/Dockerfile.monolith index 7fd25674b..0d2a141ad 100644 --- a/build/docker/Dockerfile.monolith +++ b/build/docker/Dockerfile.monolith @@ -13,6 +13,10 @@ RUN go build -trimpath -o bin/ ./cmd/create-account RUN go build -trimpath -o bin/ ./cmd/generate-keys FROM alpine:latest +LABEL org.opencontainers.image.title="Dendrite (Monolith)" +LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" +LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" +LABEL org.opencontainers.image.licenses="Apache-2.0" COPY --from=base /build/bin/* /usr/bin/ diff --git a/build/docker/Dockerfile.polylith b/build/docker/Dockerfile.polylith index 819926c4e..c266fd480 100644 --- a/build/docker/Dockerfile.polylith +++ b/build/docker/Dockerfile.polylith @@ -13,6 +13,10 @@ RUN go build -trimpath -o bin/ ./cmd/create-account RUN go build -trimpath -o bin/ ./cmd/generate-keys FROM alpine:latest +LABEL org.opencontainers.image.title="Dendrite (Polylith)" +LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" +LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" +LABEL org.opencontainers.image.licenses="Apache-2.0" COPY --from=base /build/bin/* /usr/bin/ From 31a3c1268204314d830e85b51214fd6a827dbee7 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 24 Mar 2022 15:50:30 +0000 Subject: [PATCH 14/55] Allow manual Docker tasks in GHA --- .github/workflows/docker.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 129a9f0a1..eaeb52d44 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -9,6 +9,7 @@ on: workflows: [Dendrite] types: [completed] branches: [main] + workflow_dispatch: # A build was manually requested env: DOCKER_NAMESPACE: matrixdotorg @@ -44,8 +45,8 @@ jobs: - name: Build monolith image if: >- - github.event_name == 'workflow_run' && - github.event.workflow_run.conclusion == 'success' + (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') || + github.event_name == 'workflow_dispatch' id: docker_build_monolith uses: docker/build-push-action@v2 with: @@ -56,8 +57,8 @@ jobs: platforms: ${{ env.PLATFORMS }} push: true tags: | - ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:main - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:main + ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} + ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} - name: Build release monolith image if: github.event_name == 'release' # Only for GitHub releases @@ -103,8 +104,8 @@ jobs: - name: Build polylith image if: >- - github.event_name == 'workflow_run' && - github.event.workflow_run.conclusion == 'success' + (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') || + github.event_name == 'workflow_dispatch' id: docker_build_polylith uses: docker/build-push-action@v2 with: @@ -115,8 +116,8 @@ jobs: platforms: ${{ env.PLATFORMS }} push: true tags: | - ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:main - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:main + ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} + ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} - name: Build release polylith image if: github.event_name == 'release' # Only for GitHub releases From 398aae112cd4d139b60e96fc81e0463cce45fcaf Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 24 Mar 2022 16:22:39 +0000 Subject: [PATCH 15/55] Fix Docker flow --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index eaeb52d44..7bcefb9d9 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -62,7 +62,7 @@ jobs: - name: Build release monolith image if: github.event_name == 'release' # Only for GitHub releases - id: docker_build_monolith + id: docker_build_monolith_release uses: docker/build-push-action@v2 with: cache-from: type=gha @@ -121,7 +121,7 @@ jobs: - name: Build release polylith image if: github.event_name == 'release' # Only for GitHub releases - id: docker_build_polylith + id: docker_build_polylith_release uses: docker/build-push-action@v2 with: cache-from: type=gha From 9a727416eb33d2b80e65b331abccd369ded53d3a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 24 Mar 2022 17:08:17 +0000 Subject: [PATCH 16/55] Use `github.actor` for GHCR upload --- .github/workflows/docker.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 7bcefb9d9..2c5d9e5e6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,6 +1,6 @@ # Based on https://github.com/docker/build-push-action -name: "Docker Hub" +name: "Docker" on: release: # A GitHub release was published @@ -40,7 +40,7 @@ jobs: uses: docker/login-action@v1 with: registry: ghcr.io - username: ${{ github.repository_owner }} + username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build monolith image @@ -99,7 +99,7 @@ jobs: uses: docker/login-action@v1 with: registry: ghcr.io - username: ${{ github.repository_owner }} + username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build polylith image From 8e76523b04e3ebc9546f2b019e86dcd3b516be5a Mon Sep 17 00:00:00 2001 From: S7evinK <2353100+S7evinK@users.noreply.github.com> Date: Thu, 24 Mar 2022 22:13:19 +0100 Subject: [PATCH 17/55] Update database when rejecting federated invite (#2300) * Actually set the DB entry to "leave" * Try to rollback in case of error * Fix linter issue --- roomserver/internal/perform/perform_leave.go | 24 +++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/roomserver/internal/perform/perform_leave.go b/roomserver/internal/perform/perform_leave.go index 1e5fb9f1f..5b4cd3c6f 100644 --- a/roomserver/internal/perform/perform_leave.go +++ b/roomserver/internal/perform/perform_leave.go @@ -212,12 +212,34 @@ func (r *Leaver) performFederatedRejectInvite( ServerNames: []gomatrixserverlib.ServerName{domain}, } leaveRes := fsAPI.PerformLeaveResponse{} - if err := r.FSAPI.PerformLeave(ctx, &leaveReq, &leaveRes); err != nil { + if err = r.FSAPI.PerformLeave(ctx, &leaveReq, &leaveRes); err != nil { // failures in PerformLeave should NEVER stop us from telling other components like the // sync API that the invite was withdrawn. Otherwise we can end up with stuck invites. util.GetLogger(ctx).WithError(err).Errorf("failed to PerformLeave, still retiring invite event") } + info, err := r.DB.RoomInfo(ctx, req.RoomID) + if err != nil { + util.GetLogger(ctx).WithError(err).Errorf("failed to get RoomInfo, still retiring invite event") + } + + updater, err := r.DB.MembershipUpdater(ctx, req.RoomID, req.UserID, true, info.RoomVersion) + if err != nil { + util.GetLogger(ctx).WithError(err).Errorf("failed to get MembershipUpdater, still retiring invite event") + } + if updater != nil { + if _, err = updater.SetToLeave(req.UserID, eventID); err != nil { + util.GetLogger(ctx).WithError(err).Errorf("failed to set membership to leave, still retiring invite event") + if err = updater.Rollback(); err != nil { + util.GetLogger(ctx).WithError(err).Errorf("failed to rollback membership leave, still retiring invite event") + } + } else { + if err = updater.Commit(); err != nil { + util.GetLogger(ctx).WithError(err).Errorf("failed to commit membership update, still retiring invite event") + } + } + } + // Withdraw the invite, so that the sync API etc are // notified that we rejected it. return []api.OutputEvent{ From f2e550efd832662e6c032bebfef4a68da0b4c8ee Mon Sep 17 00:00:00 2001 From: S7evinK <2353100+S7evinK@users.noreply.github.com> Date: Thu, 24 Mar 2022 22:45:44 +0100 Subject: [PATCH 18/55] Refactor appservice & client API to use userapi internal (#2290) * Refactor user api internal * Refactor clientapi to use internal userapi * Use internal userapi instead of user DB directly * Remove AccountDB dependency * Fix linter issues Co-authored-by: Neil Alexander --- appservice/api/query.go | 26 ++-- clientapi/auth/login.go | 4 +- clientapi/auth/login_test.go | 31 ++--- clientapi/auth/password.go | 33 +++-- clientapi/auth/user_interactive.go | 4 +- clientapi/auth/user_interactive_test.go | 20 ++- clientapi/clientapi.go | 4 +- clientapi/routing/createroom.go | 11 +- clientapi/routing/deactivate.go | 4 +- clientapi/routing/joinroom.go | 32 ++--- clientapi/routing/key_crosssigning.go | 5 +- clientapi/routing/login.go | 5 +- clientapi/routing/membership.go | 39 +++--- clientapi/routing/password.go | 4 +- clientapi/routing/peekroom.go | 3 - clientapi/routing/profile.go | 73 ++++++---- clientapi/routing/register.go | 37 ++--- clientapi/routing/routing.go | 58 ++++---- clientapi/routing/sendtyping.go | 3 +- clientapi/routing/server_notices.go | 15 ++- clientapi/routing/threepid.go | 40 ++++-- clientapi/threepid/invites.go | 16 ++- .../personalities/clientapi.go | 3 +- setup/monolith.go | 2 +- userapi/api/api.go | 93 ++++++++++++- userapi/api/api_trace.go | 54 ++++++++ userapi/internal/api.go | 68 ++++++++++ userapi/inthttp/client.go | 127 ++++++++++++++---- userapi/inthttp/server.go | 97 +++++++++++++ userapi/storage/interface.go | 10 +- 30 files changed, 682 insertions(+), 239 deletions(-) diff --git a/appservice/api/query.go b/appservice/api/query.go index e53ad4259..cf25a9616 100644 --- a/appservice/api/query.go +++ b/appservice/api/query.go @@ -19,11 +19,10 @@ package api import ( "context" - "database/sql" "errors" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - userdb "github.com/matrix-org/dendrite/userapi/storage" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" ) @@ -85,7 +84,7 @@ func RetrieveUserProfile( ctx context.Context, userID string, asAPI AppServiceQueryAPI, - accountDB userdb.Database, + profileAPI userapi.UserProfileAPI, ) (*authtypes.Profile, error) { localpart, _, err := gomatrixserverlib.SplitID('@', userID) if err != nil { @@ -93,10 +92,17 @@ func RetrieveUserProfile( } // Try to query the user from the local database - profile, err := accountDB.GetProfileByLocalpart(ctx, localpart) - if err != nil && err != sql.ErrNoRows { + res := &userapi.QueryProfileResponse{} + err = profileAPI.QueryProfile(ctx, &userapi.QueryProfileRequest{UserID: userID}, res) + if err != nil { return nil, err - } else if profile != nil { + } + profile := &authtypes.Profile{ + Localpart: localpart, + DisplayName: res.DisplayName, + AvatarURL: res.AvatarURL, + } + if res.UserExists { return profile, nil } @@ -113,11 +119,15 @@ func RetrieveUserProfile( } // Try to query the user from the local database again - profile, err = accountDB.GetProfileByLocalpart(ctx, localpart) + err = profileAPI.QueryProfile(ctx, &userapi.QueryProfileRequest{UserID: userID}, res) if err != nil { return nil, err } // profile should not be nil at this point - return profile, nil + return &authtypes.Profile{ + Localpart: localpart, + DisplayName: res.DisplayName, + AvatarURL: res.AvatarURL, + }, nil } diff --git a/clientapi/auth/login.go b/clientapi/auth/login.go index 1c14c6fbd..020731c9f 100644 --- a/clientapi/auth/login.go +++ b/clientapi/auth/login.go @@ -33,7 +33,7 @@ import ( // called after authorization has completed, with the result of the authorization. // If the final return value is non-nil, an error occurred and the cleanup function // is nil. -func LoginFromJSONReader(ctx context.Context, r io.Reader, accountDB AccountDatabase, userAPI UserInternalAPIForLogin, cfg *config.ClientAPI) (*Login, LoginCleanupFunc, *util.JSONResponse) { +func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.UserAccountAPI, userAPI UserInternalAPIForLogin, cfg *config.ClientAPI) (*Login, LoginCleanupFunc, *util.JSONResponse) { reqBytes, err := ioutil.ReadAll(r) if err != nil { err := &util.JSONResponse{ @@ -58,7 +58,7 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, accountDB AccountData switch header.Type { case authtypes.LoginTypePassword: typ = &LoginTypePassword{ - GetAccountByPassword: accountDB.GetAccountByPassword, + GetAccountByPassword: useraccountAPI.QueryAccountByPassword, Config: cfg, } case authtypes.LoginTypeToken: diff --git a/clientapi/auth/login_test.go b/clientapi/auth/login_test.go index e295f8f07..d401469c1 100644 --- a/clientapi/auth/login_test.go +++ b/clientapi/auth/login_test.go @@ -16,7 +16,6 @@ package auth import ( "context" - "database/sql" "net/http" "reflect" "strings" @@ -64,14 +63,13 @@ func TestLoginFromJSONReader(t *testing.T) { } for _, tst := range tsts { t.Run(tst.Name, func(t *testing.T) { - var accountDB fakeAccountDB var userAPI fakeUserInternalAPI cfg := &config.ClientAPI{ Matrix: &config.Global{ ServerName: serverName, }, } - login, cleanup, err := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &accountDB, &userAPI, cfg) + login, cleanup, err := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg) if err != nil { t.Fatalf("LoginFromJSONReader failed: %+v", err) } @@ -143,14 +141,13 @@ func TestBadLoginFromJSONReader(t *testing.T) { } for _, tst := range tsts { t.Run(tst.Name, func(t *testing.T) { - var accountDB fakeAccountDB var userAPI fakeUserInternalAPI cfg := &config.ClientAPI{ Matrix: &config.Global{ ServerName: serverName, }, } - _, cleanup, errRes := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &accountDB, &userAPI, cfg) + _, cleanup, errRes := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg) if errRes == nil { cleanup(ctx, nil) t.Fatalf("LoginFromJSONReader err: got %+v, want code %q", errRes, tst.WantErrCode) @@ -161,24 +158,22 @@ func TestBadLoginFromJSONReader(t *testing.T) { } } -type fakeAccountDB struct { - AccountDatabase -} - -func (*fakeAccountDB) GetAccountByPassword(ctx context.Context, localpart, password string) (*uapi.Account, error) { - if password == "invalidpassword" { - return nil, sql.ErrNoRows - } - - return &uapi.Account{}, nil -} - type fakeUserInternalAPI struct { UserInternalAPIForLogin - + uapi.UserAccountAPI DeletedTokens []string } +func (ua *fakeUserInternalAPI) QueryAccountByPassword(ctx context.Context, req *uapi.QueryAccountByPasswordRequest, res *uapi.QueryAccountByPasswordResponse) error { + if req.PlaintextPassword == "invalidpassword" { + res.Account = nil + return nil + } + res.Exists = true + res.Account = &uapi.Account{} + return nil +} + func (ua *fakeUserInternalAPI) PerformLoginTokenDeletion(ctx context.Context, req *uapi.PerformLoginTokenDeletionRequest, res *uapi.PerformLoginTokenDeletionResponse) error { ua.DeletedTokens = append(ua.DeletedTokens, req.Token) return nil diff --git a/clientapi/auth/password.go b/clientapi/auth/password.go index 046b36f0b..bcb4ca97b 100644 --- a/clientapi/auth/password.go +++ b/clientapi/auth/password.go @@ -16,7 +16,6 @@ package auth import ( "context" - "database/sql" "net/http" "strings" @@ -29,7 +28,7 @@ import ( "github.com/matrix-org/util" ) -type GetAccountByPassword func(ctx context.Context, localpart, password string) (*api.Account, error) +type GetAccountByPassword func(ctx context.Context, req *api.QueryAccountByPasswordRequest, res *api.QueryAccountByPasswordResponse) error type PasswordRequest struct { Login @@ -77,19 +76,33 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, } } // Squash username to all lowercase letters - _, err = t.GetAccountByPassword(ctx, strings.ToLower(localpart), r.Password) + res := &api.QueryAccountByPasswordResponse{} + err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{Localpart: strings.ToLower(localpart), PlaintextPassword: r.Password}, res) if err != nil { - if err == sql.ErrNoRows { - _, err = t.GetAccountByPassword(ctx, localpart, r.Password) - if err == nil { - return &r.Login, nil + return nil, &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: jsonerror.Unknown("unable to fetch account by password"), + } + } + + if !res.Exists { + err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{ + Localpart: localpart, + PlaintextPassword: r.Password, + }, res) + if err != nil { + return nil, &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: jsonerror.Unknown("unable to fetch account by password"), } } // Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows // but that would leak the existence of the user. - return nil, &util.JSONResponse{ - Code: http.StatusForbidden, - JSON: jsonerror.Forbidden("The username or password was incorrect or the account does not exist."), + if !res.Exists { + return nil, &util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("The username or password was incorrect or the account does not exist."), + } } } return &r.Login, nil diff --git a/clientapi/auth/user_interactive.go b/clientapi/auth/user_interactive.go index 4db75809f..22c430f97 100644 --- a/clientapi/auth/user_interactive.go +++ b/clientapi/auth/user_interactive.go @@ -110,9 +110,9 @@ type UserInteractive struct { Sessions map[string][]string } -func NewUserInteractive(accountDB AccountDatabase, cfg *config.ClientAPI) *UserInteractive { +func NewUserInteractive(userAccountAPI api.UserAccountAPI, cfg *config.ClientAPI) *UserInteractive { typePassword := &LoginTypePassword{ - GetAccountByPassword: accountDB.GetAccountByPassword, + GetAccountByPassword: userAccountAPI.QueryAccountByPassword, Config: cfg, } return &UserInteractive{ diff --git a/clientapi/auth/user_interactive_test.go b/clientapi/auth/user_interactive_test.go index 76d161a74..a4b4587a3 100644 --- a/clientapi/auth/user_interactive_test.go +++ b/clientapi/auth/user_interactive_test.go @@ -25,15 +25,25 @@ var ( ) type fakeAccountDatabase struct { - AccountDatabase + api.UserAccountAPI } -func (*fakeAccountDatabase) GetAccountByPassword(ctx context.Context, localpart, plaintextPassword string) (*api.Account, error) { - acc, ok := lookup[localpart+" "+plaintextPassword] +func (d *fakeAccountDatabase) PerformPasswordUpdate(ctx context.Context, req *api.PerformPasswordUpdateRequest, res *api.PerformPasswordUpdateResponse) error { + return nil +} + +func (d *fakeAccountDatabase) PerformAccountDeactivation(ctx context.Context, req *api.PerformAccountDeactivationRequest, res *api.PerformAccountDeactivationResponse) error { + return nil +} + +func (d *fakeAccountDatabase) QueryAccountByPassword(ctx context.Context, req *api.QueryAccountByPasswordRequest, res *api.QueryAccountByPasswordResponse) error { + acc, ok := lookup[req.Localpart+" "+req.PlaintextPassword] if !ok { - return nil, fmt.Errorf("unknown user/password") + return fmt.Errorf("unknown user/password") } - return acc, nil + res.Account = acc + res.Exists = true + return nil } func setup() *UserInteractive { diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 4550343c7..31a53a706 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -29,7 +29,6 @@ import ( "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" userapi "github.com/matrix-org/dendrite/userapi/api" - userdb "github.com/matrix-org/dendrite/userapi/storage" "github.com/matrix-org/gomatrixserverlib" ) @@ -39,7 +38,6 @@ func AddPublicRoutes( router *mux.Router, synapseAdminRouter *mux.Router, cfg *config.ClientAPI, - accountsDB userdb.Database, federation *gomatrixserverlib.FederationClient, rsAPI roomserverAPI.RoomserverInternalAPI, eduInputAPI eduServerAPI.EDUServerInputAPI, @@ -60,7 +58,7 @@ func AddPublicRoutes( routing.Setup( router, synapseAdminRouter, cfg, eduInputAPI, rsAPI, asAPI, - accountsDB, userAPI, federation, + userAPI, federation, syncProducer, transactionsCache, fsAPI, keyAPI, extRoomsProvider, mscCfg, ) diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index fcacc76c0..4976b3e50 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -31,7 +31,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/setup/config" - userdb "github.com/matrix-org/dendrite/userapi/storage" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" log "github.com/sirupsen/logrus" @@ -138,7 +137,7 @@ type fledglingEvent struct { func CreateRoom( req *http.Request, device *api.Device, cfg *config.ClientAPI, - accountDB userdb.Database, rsAPI roomserverAPI.RoomserverInternalAPI, + profileAPI api.UserProfileAPI, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { var r createRoomRequest @@ -156,7 +155,7 @@ func CreateRoom( JSON: jsonerror.InvalidArgumentValue(err.Error()), } } - return createRoom(req.Context(), r, device, cfg, accountDB, rsAPI, asAPI, evTime) + return createRoom(req.Context(), r, device, cfg, profileAPI, rsAPI, asAPI, evTime) } // createRoom implements /createRoom @@ -165,7 +164,7 @@ func createRoom( ctx context.Context, r createRoomRequest, device *api.Device, cfg *config.ClientAPI, - accountDB userdb.Database, rsAPI roomserverAPI.RoomserverInternalAPI, + profileAPI api.UserProfileAPI, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, evTime time.Time, ) util.JSONResponse { @@ -201,7 +200,7 @@ func createRoom( "roomVersion": roomVersion, }).Info("Creating new room") - profile, err := appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, accountDB) + profile, err := appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, profileAPI) if err != nil { util.GetLogger(ctx).WithError(err).Error("appserviceAPI.RetrieveUserProfile failed") return jsonerror.InternalServerError() @@ -520,7 +519,7 @@ func createRoom( for _, invitee := range r.Invite { // Build the invite event. inviteEvent, err := buildMembershipEvent( - ctx, invitee, "", accountDB, device, gomatrixserverlib.Invite, + ctx, invitee, "", profileAPI, device, gomatrixserverlib.Invite, roomID, true, cfg, evTime, rsAPI, asAPI, ) if err != nil { diff --git a/clientapi/routing/deactivate.go b/clientapi/routing/deactivate.go index 9e3b0bfb3..da1b6dcf9 100644 --- a/clientapi/routing/deactivate.go +++ b/clientapi/routing/deactivate.go @@ -15,7 +15,7 @@ import ( func Deactivate( req *http.Request, userInteractiveAuth *auth.UserInteractive, - userAPI api.UserInternalAPI, + accountAPI api.UserAccountAPI, deviceAPI *api.Device, ) util.JSONResponse { ctx := req.Context() @@ -40,7 +40,7 @@ func Deactivate( } var res api.PerformAccountDeactivationResponse - err = userAPI.PerformAccountDeactivation(ctx, &api.PerformAccountDeactivationRequest{ + err = accountAPI.PerformAccountDeactivation(ctx, &api.PerformAccountDeactivationRequest{ Localpart: localpart, }, &res) if err != nil { diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index d30a87a57..dc15f4bda 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -18,12 +18,10 @@ import ( "net/http" "time" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/userapi/api" - userdb "github.com/matrix-org/dendrite/userapi/storage" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -32,7 +30,7 @@ func JoinRoomByIDOrAlias( req *http.Request, device *api.Device, rsAPI roomserverAPI.RoomserverInternalAPI, - accountDB userdb.Database, + profileAPI api.UserProfileAPI, roomIDOrAlias string, ) util.JSONResponse { // Prepare to ask the roomserver to perform the room join. @@ -60,19 +58,23 @@ func JoinRoomByIDOrAlias( _ = httputil.UnmarshalJSONRequest(req, &joinReq.Content) // Work out our localpart for the client profile request. - localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") - } else { - // Request our profile content to populate the request content with. - var profile *authtypes.Profile - profile, err = accountDB.GetProfileByLocalpart(req.Context(), localpart) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetProfileByLocalpart failed") - } else { - joinReq.Content["displayname"] = profile.DisplayName - joinReq.Content["avatar_url"] = profile.AvatarURL + + // Request our profile content to populate the request content with. + res := &api.QueryProfileResponse{} + err := profileAPI.QueryProfile(req.Context(), &api.QueryProfileRequest{UserID: device.UserID}, res) + if err != nil || !res.UserExists { + if !res.UserExists { + util.GetLogger(req.Context()).Error("Unable to query user profile, no profile found.") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: jsonerror.Unknown("Unable to query user profile, no profile found."), + } } + + util.GetLogger(req.Context()).WithError(err).Error("UserProfileAPI.QueryProfile failed") + } else { + joinReq.Content["displayname"] = res.DisplayName + joinReq.Content["avatar_url"] = res.AvatarURL } // Ask the roomserver to perform the join. diff --git a/clientapi/routing/key_crosssigning.go b/clientapi/routing/key_crosssigning.go index 4426b7fdc..c73e0a10d 100644 --- a/clientapi/routing/key_crosssigning.go +++ b/clientapi/routing/key_crosssigning.go @@ -24,7 +24,6 @@ import ( "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" - userdb "github.com/matrix-org/dendrite/userapi/storage" "github.com/matrix-org/util" ) @@ -36,7 +35,7 @@ type crossSigningRequest struct { func UploadCrossSigningDeviceKeys( req *http.Request, userInteractiveAuth *auth.UserInteractive, keyserverAPI api.KeyInternalAPI, device *userapi.Device, - accountDB userdb.Database, cfg *config.ClientAPI, + accountAPI userapi.UserAccountAPI, cfg *config.ClientAPI, ) util.JSONResponse { uploadReq := &crossSigningRequest{} uploadRes := &api.PerformUploadDeviceKeysResponse{} @@ -64,7 +63,7 @@ func UploadCrossSigningDeviceKeys( } } typePassword := auth.LoginTypePassword{ - GetAccountByPassword: accountDB.GetAccountByPassword, + GetAccountByPassword: accountAPI.QueryAccountByPassword, Config: cfg, } if _, authErr := typePassword.Login(req.Context(), &uploadReq.Auth.PasswordRequest); authErr != nil { diff --git a/clientapi/routing/login.go b/clientapi/routing/login.go index ec5c998be..2329df504 100644 --- a/clientapi/routing/login.go +++ b/clientapi/routing/login.go @@ -23,7 +23,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" - userdb "github.com/matrix-org/dendrite/userapi/storage" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -54,7 +53,7 @@ func passwordLogin() flows { // Login implements GET and POST /login func Login( - req *http.Request, accountDB userdb.Database, userAPI userapi.UserInternalAPI, + req *http.Request, userAPI userapi.UserInternalAPI, cfg *config.ClientAPI, ) util.JSONResponse { if req.Method == http.MethodGet { @@ -64,7 +63,7 @@ func Login( JSON: passwordLogin(), } } else if req.Method == http.MethodPost { - login, cleanup, authErr := auth.LoginFromJSONReader(req.Context(), req.Body, accountDB, userAPI, cfg) + login, cleanup, authErr := auth.LoginFromJSONReader(req.Context(), req.Body, userAPI, userAPI, cfg) if authErr != nil { return *authErr } diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index ffe8da136..df8447b14 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -30,7 +30,6 @@ import ( roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" - userdb "github.com/matrix-org/dendrite/userapi/storage" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -39,7 +38,7 @@ import ( var errMissingUserID = errors.New("'user_id' must be supplied") func SendBan( - req *http.Request, accountDB userdb.Database, device *userapi.Device, + req *http.Request, profileAPI userapi.UserProfileAPI, device *userapi.Device, roomID string, cfg *config.ClientAPI, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { @@ -78,16 +77,16 @@ func SendBan( } } - return sendMembership(req.Context(), accountDB, device, roomID, "ban", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI) + return sendMembership(req.Context(), profileAPI, device, roomID, "ban", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI) } -func sendMembership(ctx context.Context, accountDB userdb.Database, device *userapi.Device, +func sendMembership(ctx context.Context, profileAPI userapi.UserProfileAPI, device *userapi.Device, roomID, membership, reason string, cfg *config.ClientAPI, targetUserID string, evTime time.Time, roomVer gomatrixserverlib.RoomVersion, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI) util.JSONResponse { event, err := buildMembershipEvent( - ctx, targetUserID, reason, accountDB, device, membership, + ctx, targetUserID, reason, profileAPI, device, membership, roomID, false, cfg, evTime, rsAPI, asAPI, ) if err == errMissingUserID { @@ -125,7 +124,7 @@ func sendMembership(ctx context.Context, accountDB userdb.Database, device *user } func SendKick( - req *http.Request, accountDB userdb.Database, device *userapi.Device, + req *http.Request, profileAPI userapi.UserProfileAPI, device *userapi.Device, roomID string, cfg *config.ClientAPI, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { @@ -161,11 +160,11 @@ func SendKick( } } // TODO: should we be using SendLeave instead? - return sendMembership(req.Context(), accountDB, device, roomID, "leave", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI) + return sendMembership(req.Context(), profileAPI, device, roomID, "leave", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI) } func SendUnban( - req *http.Request, accountDB userdb.Database, device *userapi.Device, + req *http.Request, profileAPI userapi.UserProfileAPI, device *userapi.Device, roomID string, cfg *config.ClientAPI, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { @@ -196,11 +195,11 @@ func SendUnban( } } // TODO: should we be using SendLeave instead? - return sendMembership(req.Context(), accountDB, device, roomID, "leave", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI) + return sendMembership(req.Context(), profileAPI, device, roomID, "leave", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI) } func SendInvite( - req *http.Request, accountDB userdb.Database, device *userapi.Device, + req *http.Request, profileAPI userapi.UserProfileAPI, device *userapi.Device, roomID string, cfg *config.ClientAPI, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { @@ -210,7 +209,7 @@ func SendInvite( } inviteStored, jsonErrResp := checkAndProcessThreepid( - req, device, body, cfg, rsAPI, accountDB, roomID, evTime, + req, device, body, cfg, rsAPI, profileAPI, roomID, evTime, ) if jsonErrResp != nil { return *jsonErrResp @@ -227,14 +226,14 @@ func SendInvite( } // We already received the return value, so no need to check for an error here. - response, _ := sendInvite(req.Context(), accountDB, device, roomID, body.UserID, body.Reason, cfg, rsAPI, asAPI, evTime) + response, _ := sendInvite(req.Context(), profileAPI, device, roomID, body.UserID, body.Reason, cfg, rsAPI, asAPI, evTime) return response } // sendInvite sends an invitation to a user. Returns a JSONResponse and an error func sendInvite( ctx context.Context, - accountDB userdb.Database, + profileAPI userapi.UserProfileAPI, device *userapi.Device, roomID, userID, reason string, cfg *config.ClientAPI, @@ -242,7 +241,7 @@ func sendInvite( asAPI appserviceAPI.AppServiceQueryAPI, evTime time.Time, ) (util.JSONResponse, error) { event, err := buildMembershipEvent( - ctx, userID, reason, accountDB, device, "invite", + ctx, userID, reason, profileAPI, device, "invite", roomID, false, cfg, evTime, rsAPI, asAPI, ) if err == errMissingUserID { @@ -286,13 +285,13 @@ func sendInvite( func buildMembershipEvent( ctx context.Context, - targetUserID, reason string, accountDB userdb.Database, + targetUserID, reason string, profileAPI userapi.UserProfileAPI, device *userapi.Device, membership, roomID string, isDirect bool, cfg *config.ClientAPI, evTime time.Time, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) (*gomatrixserverlib.HeaderedEvent, error) { - profile, err := loadProfile(ctx, targetUserID, cfg, accountDB, asAPI) + profile, err := loadProfile(ctx, targetUserID, cfg, profileAPI, asAPI) if err != nil { return nil, err } @@ -327,7 +326,7 @@ func loadProfile( ctx context.Context, userID string, cfg *config.ClientAPI, - accountDB userdb.Database, + profileAPI userapi.UserProfileAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) (*authtypes.Profile, error) { _, serverName, err := gomatrixserverlib.SplitID('@', userID) @@ -337,7 +336,7 @@ func loadProfile( var profile *authtypes.Profile if serverName == cfg.Matrix.ServerName { - profile, err = appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, accountDB) + profile, err = appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, profileAPI) } else { profile = &authtypes.Profile{} } @@ -381,13 +380,13 @@ func checkAndProcessThreepid( body *threepid.MembershipRequest, cfg *config.ClientAPI, rsAPI roomserverAPI.RoomserverInternalAPI, - accountDB userdb.Database, + profileAPI userapi.UserProfileAPI, roomID string, evTime time.Time, ) (inviteStored bool, errRes *util.JSONResponse) { inviteStored, err := threepid.CheckAndProcessInvite( - req.Context(), device, body, cfg, rsAPI, accountDB, + req.Context(), device, body, cfg, rsAPI, profileAPI, roomID, evTime, ) if err == threepid.ErrMissingParameter { diff --git a/clientapi/routing/password.go b/clientapi/routing/password.go index c63412d08..08ce1ffa1 100644 --- a/clientapi/routing/password.go +++ b/clientapi/routing/password.go @@ -9,7 +9,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/userapi/api" - userdb "github.com/matrix-org/dendrite/userapi/storage" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/sirupsen/logrus" @@ -30,7 +29,6 @@ type newPasswordAuth struct { func Password( req *http.Request, userAPI api.UserInternalAPI, - accountDB userdb.Database, device *api.Device, cfg *config.ClientAPI, ) util.JSONResponse { @@ -74,7 +72,7 @@ func Password( // Check if the existing password is correct. typePassword := auth.LoginTypePassword{ - GetAccountByPassword: accountDB.GetAccountByPassword, + GetAccountByPassword: userAPI.QueryAccountByPassword, Config: cfg, } if _, authErr := typePassword.Login(req.Context(), &r.Auth.PasswordRequest); authErr != nil { diff --git a/clientapi/routing/peekroom.go b/clientapi/routing/peekroom.go index 8f89e97f4..41d1ff004 100644 --- a/clientapi/routing/peekroom.go +++ b/clientapi/routing/peekroom.go @@ -19,7 +19,6 @@ import ( roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/userapi/api" - userdb "github.com/matrix-org/dendrite/userapi/storage" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -28,7 +27,6 @@ func PeekRoomByIDOrAlias( req *http.Request, device *api.Device, rsAPI roomserverAPI.RoomserverInternalAPI, - accountDB userdb.Database, roomIDOrAlias string, ) util.JSONResponse { // if this is a remote roomIDOrAlias, we have to ask the roomserver (or federation sender?) to @@ -82,7 +80,6 @@ func UnpeekRoomByID( req *http.Request, device *api.Device, rsAPI roomserverAPI.RoomserverInternalAPI, - accountDB userdb.Database, roomID string, ) util.JSONResponse { unpeekReq := roomserverAPI.PerformUnpeekRequest{ diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index 4aab03714..dd1da1806 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -27,7 +27,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" - userdb "github.com/matrix-org/dendrite/userapi/storage" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrix" @@ -36,12 +35,12 @@ import ( // GetProfile implements GET /profile/{userID} func GetProfile( - req *http.Request, accountDB userdb.Database, cfg *config.ClientAPI, + req *http.Request, profileAPI userapi.UserProfileAPI, cfg *config.ClientAPI, userID string, asAPI appserviceAPI.AppServiceQueryAPI, federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { - profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation) + profile, err := getProfile(req.Context(), profileAPI, cfg, userID, asAPI, federation) if err != nil { if err == eventutil.ErrProfileNoExists { return util.JSONResponse{ @@ -65,11 +64,11 @@ func GetProfile( // GetAvatarURL implements GET /profile/{userID}/avatar_url func GetAvatarURL( - req *http.Request, accountDB userdb.Database, cfg *config.ClientAPI, + req *http.Request, profileAPI userapi.UserProfileAPI, cfg *config.ClientAPI, userID string, asAPI appserviceAPI.AppServiceQueryAPI, federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { - profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation) + profile, err := getProfile(req.Context(), profileAPI, cfg, userID, asAPI, federation) if err != nil { if err == eventutil.ErrProfileNoExists { return util.JSONResponse{ @@ -92,7 +91,7 @@ func GetAvatarURL( // SetAvatarURL implements PUT /profile/{userID}/avatar_url func SetAvatarURL( - req *http.Request, accountDB userdb.Database, + req *http.Request, profileAPI userapi.UserProfileAPI, device *userapi.Device, userID string, cfg *config.ClientAPI, rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { if userID != device.UserID { @@ -127,22 +126,34 @@ func SetAvatarURL( } } - oldProfile, err := accountDB.GetProfileByLocalpart(req.Context(), localpart) + res := &userapi.QueryProfileResponse{} + err = profileAPI.QueryProfile(req.Context(), &userapi.QueryProfileRequest{ + UserID: userID, + }, res) if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetProfileByLocalpart failed") + util.GetLogger(req.Context()).WithError(err).Error("profileAPI.QueryProfile failed") + return jsonerror.InternalServerError() + } + oldProfile := &authtypes.Profile{ + Localpart: localpart, + DisplayName: res.DisplayName, + AvatarURL: res.AvatarURL, + } + + setRes := &userapi.PerformSetAvatarURLResponse{} + if err = profileAPI.SetAvatarURL(req.Context(), &userapi.PerformSetAvatarURLRequest{ + Localpart: localpart, + AvatarURL: r.AvatarURL, + }, setRes); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("profileAPI.SetAvatarURL failed") return jsonerror.InternalServerError() } - if err = accountDB.SetAvatarURL(req.Context(), localpart, r.AvatarURL); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("accountDB.SetAvatarURL failed") - return jsonerror.InternalServerError() - } - - var res api.QueryRoomsForUserResponse + var roomsRes api.QueryRoomsForUserResponse err = rsAPI.QueryRoomsForUser(req.Context(), &api.QueryRoomsForUserRequest{ UserID: device.UserID, WantMembership: "join", - }, &res) + }, &roomsRes) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("QueryRoomsForUser failed") return jsonerror.InternalServerError() @@ -155,7 +166,7 @@ func SetAvatarURL( } events, err := buildMembershipEvents( - req.Context(), res.RoomIDs, newProfile, userID, cfg, evTime, rsAPI, + req.Context(), roomsRes.RoomIDs, newProfile, userID, cfg, evTime, rsAPI, ) switch e := err.(type) { case nil: @@ -182,11 +193,11 @@ func SetAvatarURL( // GetDisplayName implements GET /profile/{userID}/displayname func GetDisplayName( - req *http.Request, accountDB userdb.Database, cfg *config.ClientAPI, + req *http.Request, profileAPI userapi.UserProfileAPI, cfg *config.ClientAPI, userID string, asAPI appserviceAPI.AppServiceQueryAPI, federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { - profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation) + profile, err := getProfile(req.Context(), profileAPI, cfg, userID, asAPI, federation) if err != nil { if err == eventutil.ErrProfileNoExists { return util.JSONResponse{ @@ -209,7 +220,7 @@ func GetDisplayName( // SetDisplayName implements PUT /profile/{userID}/displayname func SetDisplayName( - req *http.Request, accountDB userdb.Database, + req *http.Request, profileAPI userapi.UserProfileAPI, device *userapi.Device, userID string, cfg *config.ClientAPI, rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { if userID != device.UserID { @@ -244,14 +255,26 @@ func SetDisplayName( } } - oldProfile, err := accountDB.GetProfileByLocalpart(req.Context(), localpart) + pRes := &userapi.QueryProfileResponse{} + err = profileAPI.QueryProfile(req.Context(), &userapi.QueryProfileRequest{ + UserID: userID, + }, pRes) if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetProfileByLocalpart failed") + util.GetLogger(req.Context()).WithError(err).Error("profileAPI.QueryProfile failed") return jsonerror.InternalServerError() } + oldProfile := &authtypes.Profile{ + Localpart: localpart, + DisplayName: pRes.DisplayName, + AvatarURL: pRes.AvatarURL, + } - if err = accountDB.SetDisplayName(req.Context(), localpart, r.DisplayName); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("accountDB.SetDisplayName failed") + err = profileAPI.SetDisplayName(req.Context(), &userapi.PerformUpdateDisplayNameRequest{ + Localpart: localpart, + DisplayName: r.DisplayName, + }, &struct{}{}) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("profileAPI.SetDisplayName failed") return jsonerror.InternalServerError() } @@ -302,7 +325,7 @@ func SetDisplayName( // Returns an error when something goes wrong or specifically // eventutil.ErrProfileNoExists when the profile doesn't exist. func getProfile( - ctx context.Context, accountDB userdb.Database, cfg *config.ClientAPI, + ctx context.Context, profileAPI userapi.UserProfileAPI, cfg *config.ClientAPI, userID string, asAPI appserviceAPI.AppServiceQueryAPI, federation *gomatrixserverlib.FederationClient, @@ -331,7 +354,7 @@ func getProfile( }, nil } - profile, err := appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, accountDB) + profile, err := appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, profileAPI) if err != nil { return nil, err } diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index c97876594..af2e99ed0 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -44,7 +44,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/userutil" userapi "github.com/matrix-org/dendrite/userapi/api" - userdb "github.com/matrix-org/dendrite/userapi/storage" ) var ( @@ -523,8 +522,7 @@ func validateApplicationService( // http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register func Register( req *http.Request, - userAPI userapi.UserInternalAPI, - accountDB userdb.Database, + userAPI userapi.UserRegisterAPI, cfg *config.ClientAPI, ) util.JSONResponse { var r registerRequest @@ -552,13 +550,12 @@ func Register( } // Auto generate a numeric username if r.Username is empty if r.Username == "" { - id, err := accountDB.GetNewNumericLocalpart(req.Context()) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetNewNumericLocalpart failed") + res := &userapi.QueryNumericLocalpartResponse{} + if err := userAPI.QueryNumericLocalpart(req.Context(), res); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryNumericLocalpart failed") return jsonerror.InternalServerError() } - - r.Username = strconv.FormatInt(id, 10) + r.Username = strconv.FormatInt(res.ID, 10) } // Is this an appservice registration? It will be if the access @@ -606,7 +603,7 @@ func handleGuestRegistration( req *http.Request, r registerRequest, cfg *config.ClientAPI, - userAPI userapi.UserInternalAPI, + userAPI userapi.UserRegisterAPI, ) util.JSONResponse { if cfg.RegistrationDisabled || cfg.GuestsDisabled { return util.JSONResponse{ @@ -671,7 +668,7 @@ func handleRegistrationFlow( r registerRequest, sessionID string, cfg *config.ClientAPI, - userAPI userapi.UserInternalAPI, + userAPI userapi.UserRegisterAPI, accessToken string, accessTokenErr error, ) util.JSONResponse { @@ -760,7 +757,7 @@ func handleApplicationServiceRegistration( req *http.Request, r registerRequest, cfg *config.ClientAPI, - userAPI userapi.UserInternalAPI, + userAPI userapi.UserRegisterAPI, ) util.JSONResponse { // Check if we previously had issues extracting the access token from the // request. @@ -798,7 +795,7 @@ func checkAndCompleteFlow( r registerRequest, sessionID string, cfg *config.ClientAPI, - userAPI userapi.UserInternalAPI, + userAPI userapi.UserRegisterAPI, ) util.JSONResponse { if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) { // This flow was completed, registration can continue @@ -825,7 +822,7 @@ func checkAndCompleteFlow( // not all func completeRegistration( ctx context.Context, - userAPI userapi.UserInternalAPI, + userAPI userapi.UserRegisterAPI, username, password, appserviceID, ipAddr, userAgent, sessionID string, inhibitLogin eventutil.WeakBoolean, displayName, deviceID *string, @@ -991,7 +988,7 @@ type availableResponse struct { func RegisterAvailable( req *http.Request, cfg *config.ClientAPI, - accountDB userdb.Database, + registerAPI userapi.UserRegisterAPI, ) util.JSONResponse { username := req.URL.Query().Get("username") @@ -1013,14 +1010,18 @@ func RegisterAvailable( } } - availability, availabilityErr := accountDB.CheckAccountAvailability(req.Context(), username) - if availabilityErr != nil { + res := &userapi.QueryAccountAvailabilityResponse{} + err := registerAPI.QueryAccountAvailability(req.Context(), &userapi.QueryAccountAvailabilityRequest{ + Localpart: username, + }, res) + if err != nil { return util.JSONResponse{ Code: http.StatusInternalServerError, - JSON: jsonerror.Unknown("failed to check availability: " + availabilityErr.Error()), + JSON: jsonerror.Unknown("failed to check availability:" + err.Error()), } } - if !availability { + + if !res.Available { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UserInUse("Desired User ID is already taken."), diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index d22fbd809..e173831ab 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -34,7 +34,6 @@ import ( roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" - userdb "github.com/matrix-org/dendrite/userapi/storage" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/sirupsen/logrus" @@ -51,7 +50,6 @@ func Setup( eduAPI eduServerAPI.EDUServerInputAPI, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, - accountDB userdb.Database, userAPI userapi.UserInternalAPI, federation *gomatrixserverlib.FederationClient, syncProducer *producers.SyncAPIProducer, @@ -62,7 +60,7 @@ func Setup( mscCfg *config.MSCs, ) { rateLimits := httputil.NewRateLimits(&cfg.RateLimiting) - userInteractiveAuth := auth.NewUserInteractive(accountDB, cfg) + userInteractiveAuth := auth.NewUserInteractive(userAPI, cfg) unstableFeatures := map[string]bool{ "org.matrix.e2e_cross_signing": true, @@ -120,7 +118,7 @@ func Setup( // server notifications if cfg.Matrix.ServerNotices.Enabled { logrus.Info("Enabling server notices at /_synapse/admin/v1/send_server_notice") - serverNotificationSender, err := getSenderDevice(context.Background(), userAPI, accountDB, cfg) + serverNotificationSender, err := getSenderDevice(context.Background(), userAPI, cfg) if err != nil { logrus.WithError(err).Fatal("unable to get account for sending sending server notices") } @@ -138,7 +136,7 @@ func Setup( txnID := vars["txnID"] return SendServerNotice( req, &cfg.Matrix.ServerNotices, - cfg, userAPI, rsAPI, accountDB, asAPI, + cfg, userAPI, rsAPI, asAPI, device, serverNotificationSender, &txnID, transactionsCache, ) @@ -153,7 +151,7 @@ func Setup( } return SendServerNotice( req, &cfg.Matrix.ServerNotices, - cfg, userAPI, rsAPI, accountDB, asAPI, + cfg, userAPI, rsAPI, asAPI, device, serverNotificationSender, nil, transactionsCache, ) @@ -173,7 +171,7 @@ func Setup( v3mux.Handle("/createRoom", httputil.MakeAuthAPI("createRoom", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - return CreateRoom(req, device, cfg, accountDB, rsAPI, asAPI) + return CreateRoom(req, device, cfg, userAPI, rsAPI, asAPI) }), ).Methods(http.MethodPost, http.MethodOptions) v3mux.Handle("/join/{roomIDOrAlias}", @@ -186,7 +184,7 @@ func Setup( return util.ErrorResponse(err) } return JoinRoomByIDOrAlias( - req, device, rsAPI, accountDB, vars["roomIDOrAlias"], + req, device, rsAPI, userAPI, vars["roomIDOrAlias"], ) }), ).Methods(http.MethodPost, http.MethodOptions) @@ -202,7 +200,7 @@ func Setup( return util.ErrorResponse(err) } return PeekRoomByIDOrAlias( - req, device, rsAPI, accountDB, vars["roomIDOrAlias"], + req, device, rsAPI, vars["roomIDOrAlias"], ) }), ).Methods(http.MethodPost, http.MethodOptions) @@ -222,7 +220,7 @@ func Setup( return util.ErrorResponse(err) } return JoinRoomByIDOrAlias( - req, device, rsAPI, accountDB, vars["roomID"], + req, device, rsAPI, userAPI, vars["roomID"], ) }), ).Methods(http.MethodPost, http.MethodOptions) @@ -247,7 +245,7 @@ func Setup( return util.ErrorResponse(err) } return UnpeekRoomByID( - req, device, rsAPI, accountDB, vars["roomID"], + req, device, rsAPI, vars["roomID"], ) }), ).Methods(http.MethodPost, http.MethodOptions) @@ -257,7 +255,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SendBan(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI) + return SendBan(req, userAPI, device, vars["roomID"], cfg, rsAPI, asAPI) }), ).Methods(http.MethodPost, http.MethodOptions) v3mux.Handle("/rooms/{roomID}/invite", @@ -269,7 +267,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SendInvite(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI) + return SendInvite(req, userAPI, device, vars["roomID"], cfg, rsAPI, asAPI) }), ).Methods(http.MethodPost, http.MethodOptions) v3mux.Handle("/rooms/{roomID}/kick", @@ -278,7 +276,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SendKick(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI) + return SendKick(req, userAPI, device, vars["roomID"], cfg, rsAPI, asAPI) }), ).Methods(http.MethodPost, http.MethodOptions) v3mux.Handle("/rooms/{roomID}/unban", @@ -287,7 +285,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SendUnban(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI) + return SendUnban(req, userAPI, device, vars["roomID"], cfg, rsAPI, asAPI) }), ).Methods(http.MethodPost, http.MethodOptions) v3mux.Handle("/rooms/{roomID}/send/{eventType}", @@ -383,14 +381,14 @@ func Setup( if r := rateLimits.Limit(req); r != nil { return *r } - return Register(req, userAPI, accountDB, cfg) + return Register(req, userAPI, cfg) })).Methods(http.MethodPost, http.MethodOptions) v3mux.Handle("/register/available", httputil.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse { if r := rateLimits.Limit(req); r != nil { return *r } - return RegisterAvailable(req, cfg, accountDB) + return RegisterAvailable(req, cfg, userAPI) })).Methods(http.MethodGet, http.MethodOptions) v3mux.Handle("/directory/room/{roomAlias}", @@ -468,7 +466,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SendTyping(req, device, vars["roomID"], vars["userID"], accountDB, eduAPI, rsAPI) + return SendTyping(req, device, vars["roomID"], vars["userID"], eduAPI, rsAPI) }), ).Methods(http.MethodPut, http.MethodOptions) v3mux.Handle("/rooms/{roomID}/redact/{eventID}", @@ -529,7 +527,7 @@ func Setup( if r := rateLimits.Limit(req); r != nil { return *r } - return Password(req, userAPI, accountDB, device, cfg) + return Password(req, userAPI, device, cfg) }), ).Methods(http.MethodPost, http.MethodOptions) @@ -549,7 +547,7 @@ func Setup( if r := rateLimits.Limit(req); r != nil { return *r } - return Login(req, accountDB, userAPI, cfg) + return Login(req, userAPI, cfg) }), ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) @@ -704,7 +702,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetProfile(req, accountDB, cfg, vars["userID"], asAPI, federation) + return GetProfile(req, userAPI, cfg, vars["userID"], asAPI, federation) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -714,7 +712,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetAvatarURL(req, accountDB, cfg, vars["userID"], asAPI, federation) + return GetAvatarURL(req, userAPI, cfg, vars["userID"], asAPI, federation) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -727,7 +725,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SetAvatarURL(req, accountDB, device, vars["userID"], cfg, rsAPI) + return SetAvatarURL(req, userAPI, device, vars["userID"], cfg, rsAPI) }), ).Methods(http.MethodPut, http.MethodOptions) // Browsers use the OPTIONS HTTP method to check if the CORS policy allows @@ -739,7 +737,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetDisplayName(req, accountDB, cfg, vars["userID"], asAPI, federation) + return GetDisplayName(req, userAPI, cfg, vars["userID"], asAPI, federation) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -752,7 +750,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SetDisplayName(req, accountDB, device, vars["userID"], cfg, rsAPI) + return SetDisplayName(req, userAPI, device, vars["userID"], cfg, rsAPI) }), ).Methods(http.MethodPut, http.MethodOptions) // Browsers use the OPTIONS HTTP method to check if the CORS policy allows @@ -760,25 +758,25 @@ func Setup( v3mux.Handle("/account/3pid", httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - return GetAssociated3PIDs(req, accountDB, device) + return GetAssociated3PIDs(req, userAPI, device) }), ).Methods(http.MethodGet, http.MethodOptions) v3mux.Handle("/account/3pid", httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - return CheckAndSave3PIDAssociation(req, accountDB, device, cfg) + return CheckAndSave3PIDAssociation(req, userAPI, device, cfg) }), ).Methods(http.MethodPost, http.MethodOptions) unstableMux.Handle("/account/3pid/delete", httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - return Forget3PID(req, accountDB) + return Forget3PID(req, userAPI) }), ).Methods(http.MethodPost, http.MethodOptions) v3mux.Handle("/{path:(?:account/3pid|register)}/email/requestToken", httputil.MakeExternalAPI("account_3pid_request_token", func(req *http.Request) util.JSONResponse { - return RequestEmailToken(req, accountDB, cfg) + return RequestEmailToken(req, userAPI, cfg) }), ).Methods(http.MethodPost, http.MethodOptions) @@ -1253,7 +1251,7 @@ func Setup( // Cross-signing device keys postDeviceSigningKeys := httputil.MakeAuthAPI("post_device_signing_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - return UploadCrossSigningDeviceKeys(req, userInteractiveAuth, keyAPI, device, accountDB, cfg) + return UploadCrossSigningDeviceKeys(req, userInteractiveAuth, keyAPI, device, userAPI, cfg) }) postDeviceSigningSignatures := httputil.MakeAuthAPI("post_device_signing_signatures", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { diff --git a/clientapi/routing/sendtyping.go b/clientapi/routing/sendtyping.go index fd214b34b..abd2061a8 100644 --- a/clientapi/routing/sendtyping.go +++ b/clientapi/routing/sendtyping.go @@ -20,7 +20,6 @@ import ( "github.com/matrix-org/dendrite/eduserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" userapi "github.com/matrix-org/dendrite/userapi/api" - userdb "github.com/matrix-org/dendrite/userapi/storage" "github.com/matrix-org/util" ) @@ -33,7 +32,7 @@ type typingContentJSON struct { // sends the typing events to client API typingProducer func SendTyping( req *http.Request, device *userapi.Device, roomID string, - userID string, accountDB userdb.Database, + userID string, eduAPI api.EDUServerInputAPI, rsAPI roomserverAPI.RoomserverInternalAPI, ) util.JSONResponse { diff --git a/clientapi/routing/server_notices.go b/clientapi/routing/server_notices.go index 42a303a6b..eec3d7e38 100644 --- a/clientapi/routing/server_notices.go +++ b/clientapi/routing/server_notices.go @@ -21,7 +21,6 @@ import ( "net/http" "time" - userdb "github.com/matrix-org/dendrite/userapi/storage" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/tokens" @@ -58,7 +57,6 @@ func SendServerNotice( cfgClient *config.ClientAPI, userAPI userapi.UserInternalAPI, rsAPI api.RoomserverInternalAPI, - accountsDB userdb.Database, asAPI appserviceAPI.AppServiceQueryAPI, device *userapi.Device, senderDevice *userapi.Device, @@ -175,7 +173,7 @@ func SendServerNotice( PowerLevelContentOverride: pl, } - roomRes := createRoom(ctx, crReq, senderDevice, cfgClient, accountsDB, rsAPI, asAPI, time.Now()) + roomRes := createRoom(ctx, crReq, senderDevice, cfgClient, userAPI, rsAPI, asAPI, time.Now()) switch data := roomRes.JSON.(type) { case createRoomResponse: @@ -201,7 +199,7 @@ func SendServerNotice( // we've found a room in common, check the membership roomID = commonRooms[0] // re-invite the user - res, err := sendInvite(ctx, accountsDB, senderDevice, roomID, r.UserID, "Server notice room", cfgClient, rsAPI, asAPI, time.Now()) + res, err := sendInvite(ctx, userAPI, senderDevice, roomID, r.UserID, "Server notice room", cfgClient, rsAPI, asAPI, time.Now()) if err != nil { return res } @@ -284,7 +282,6 @@ func (r sendServerNoticeRequest) valid() (ok bool) { func getSenderDevice( ctx context.Context, userAPI userapi.UserInternalAPI, - accountDB userdb.Database, cfg *config.ClientAPI, ) (*userapi.Device, error) { var accRes userapi.PerformAccountCreationResponse @@ -299,8 +296,12 @@ func getSenderDevice( } // set the avatarurl for the user - if err = accountDB.SetAvatarURL(ctx, cfg.Matrix.ServerNotices.LocalPart, cfg.Matrix.ServerNotices.AvatarURL); err != nil { - util.GetLogger(ctx).WithError(err).Error("accountDB.SetAvatarURL failed") + res := &userapi.PerformSetAvatarURLResponse{} + if err = userAPI.SetAvatarURL(ctx, &userapi.PerformSetAvatarURLRequest{ + Localpart: cfg.Matrix.ServerNotices.LocalPart, + AvatarURL: cfg.Matrix.ServerNotices.AvatarURL, + }, res); err != nil { + util.GetLogger(ctx).WithError(err).Error("userAPI.SetAvatarURL failed") return nil, err } diff --git a/clientapi/routing/threepid.go b/clientapi/routing/threepid.go index d89b62953..a4898ca46 100644 --- a/clientapi/routing/threepid.go +++ b/clientapi/routing/threepid.go @@ -40,7 +40,7 @@ type threePIDsResponse struct { // RequestEmailToken implements: // POST /account/3pid/email/requestToken // POST /register/email/requestToken -func RequestEmailToken(req *http.Request, accountDB userdb.Database, cfg *config.ClientAPI) util.JSONResponse { +func RequestEmailToken(req *http.Request, threePIDAPI api.UserThreePIDAPI, cfg *config.ClientAPI) util.JSONResponse { var body threepid.EmailAssociationRequest if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil { return *reqErr @@ -50,13 +50,18 @@ func RequestEmailToken(req *http.Request, accountDB userdb.Database, cfg *config var err error // Check if the 3PID is already in use locally - localpart, err := accountDB.GetLocalpartForThreePID(req.Context(), body.Email, "email") + res := &api.QueryLocalpartForThreePIDResponse{} + err = threePIDAPI.QueryLocalpartForThreePID(req.Context(), &api.QueryLocalpartForThreePIDRequest{ + ThreePID: body.Email, + Medium: "email", + }, res) + if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetLocalpartForThreePID failed") + util.GetLogger(req.Context()).WithError(err).Error("threePIDAPI.QueryLocalpartForThreePID failed") return jsonerror.InternalServerError() } - if len(localpart) > 0 { + if len(res.Localpart) > 0 { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.MatrixError{ @@ -85,7 +90,7 @@ func RequestEmailToken(req *http.Request, accountDB userdb.Database, cfg *config // CheckAndSave3PIDAssociation implements POST /account/3pid func CheckAndSave3PIDAssociation( - req *http.Request, accountDB userdb.Database, device *api.Device, + req *http.Request, threePIDAPI api.UserThreePIDAPI, device *api.Device, cfg *config.ClientAPI, ) util.JSONResponse { var body threepid.EmailAssociationCheckRequest @@ -136,8 +141,12 @@ func CheckAndSave3PIDAssociation( return jsonerror.InternalServerError() } - if err = accountDB.SaveThreePIDAssociation(req.Context(), address, localpart, medium); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("accountsDB.SaveThreePIDAssociation failed") + if err = threePIDAPI.PerformSaveThreePIDAssociation(req.Context(), &api.PerformSaveThreePIDAssociationRequest{ + ThreePID: address, + Localpart: localpart, + Medium: medium, + }, &struct{}{}); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("threePIDAPI.PerformSaveThreePIDAssociation failed") return jsonerror.InternalServerError() } @@ -149,7 +158,7 @@ func CheckAndSave3PIDAssociation( // GetAssociated3PIDs implements GET /account/3pid func GetAssociated3PIDs( - req *http.Request, accountDB userdb.Database, device *api.Device, + req *http.Request, threepidAPI api.UserThreePIDAPI, device *api.Device, ) util.JSONResponse { localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) if err != nil { @@ -157,27 +166,30 @@ func GetAssociated3PIDs( return jsonerror.InternalServerError() } - threepids, err := accountDB.GetThreePIDsForLocalpart(req.Context(), localpart) + res := &api.QueryThreePIDsForLocalpartResponse{} + err = threepidAPI.QueryThreePIDsForLocalpart(req.Context(), &api.QueryThreePIDsForLocalpartRequest{ + Localpart: localpart, + }, res) if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetThreePIDsForLocalpart failed") + util.GetLogger(req.Context()).WithError(err).Error("threepidAPI.QueryThreePIDsForLocalpart failed") return jsonerror.InternalServerError() } return util.JSONResponse{ Code: http.StatusOK, - JSON: threePIDsResponse{threepids}, + JSON: threePIDsResponse{res.ThreePIDs}, } } // Forget3PID implements POST /account/3pid/delete -func Forget3PID(req *http.Request, accountDB userdb.Database) util.JSONResponse { +func Forget3PID(req *http.Request, threepidAPI api.UserThreePIDAPI) util.JSONResponse { var body authtypes.ThreePID if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil { return *reqErr } - if err := accountDB.RemoveThreePIDAssociation(req.Context(), body.Address, body.Medium); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("accountDB.RemoveThreePIDAssociation failed") + if err := threepidAPI.PerformForgetThreePID(req.Context(), &api.PerformForgetThreePIDRequest{}, &struct{}{}); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("threepidAPI.PerformForgetThreePID failed") return jsonerror.InternalServerError() } diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index 9d9a2ba7a..6b750199b 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -29,7 +29,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" - userdb "github.com/matrix-org/dendrite/userapi/storage" "github.com/matrix-org/gomatrixserverlib" ) @@ -87,7 +86,7 @@ var ( func CheckAndProcessInvite( ctx context.Context, device *userapi.Device, body *MembershipRequest, cfg *config.ClientAPI, - rsAPI api.RoomserverInternalAPI, db userdb.Database, + rsAPI api.RoomserverInternalAPI, db userapi.UserProfileAPI, roomID string, evTime time.Time, ) (inviteStoredOnIDServer bool, err error) { @@ -137,7 +136,7 @@ func CheckAndProcessInvite( // Returns an error if a check or a request failed. func queryIDServer( ctx context.Context, - db userdb.Database, cfg *config.ClientAPI, device *userapi.Device, + db userapi.UserProfileAPI, cfg *config.ClientAPI, device *userapi.Device, body *MembershipRequest, roomID string, ) (lookupRes *idServerLookupResponse, storeInviteRes *idServerStoreInviteResponse, err error) { if err = isTrusted(body.IDServer, cfg); err != nil { @@ -206,7 +205,7 @@ func queryIDServerLookup(ctx context.Context, body *MembershipRequest) (*idServe // Returns an error if the request failed to send or if the response couldn't be parsed. func queryIDServerStoreInvite( ctx context.Context, - db userdb.Database, cfg *config.ClientAPI, device *userapi.Device, + db userapi.UserProfileAPI, cfg *config.ClientAPI, device *userapi.Device, body *MembershipRequest, roomID string, ) (*idServerStoreInviteResponse, error) { // Retrieve the sender's profile to get their display name @@ -217,10 +216,17 @@ func queryIDServerStoreInvite( var profile *authtypes.Profile if serverName == cfg.Matrix.ServerName { - profile, err = db.GetProfileByLocalpart(ctx, localpart) + res := &userapi.QueryProfileResponse{} + err = db.QueryProfile(ctx, &userapi.QueryProfileRequest{UserID: device.UserID}, res) if err != nil { return nil, err } + profile = &authtypes.Profile{ + Localpart: localpart, + DisplayName: res.DisplayName, + AvatarURL: res.AvatarURL, + } + } else { profile = &authtypes.Profile{} } diff --git a/cmd/dendrite-polylith-multi/personalities/clientapi.go b/cmd/dendrite-polylith-multi/personalities/clientapi.go index 5e67acd03..a2036de35 100644 --- a/cmd/dendrite-polylith-multi/personalities/clientapi.go +++ b/cmd/dendrite-polylith-multi/personalities/clientapi.go @@ -22,7 +22,6 @@ import ( ) func ClientAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { - accountDB := base.CreateAccountsDB() federation := base.CreateFederationClient() asQuery := base.AppserviceHTTPClient() @@ -34,7 +33,7 @@ func ClientAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { clientapi.AddPublicRoutes( base.ProcessContext, base.PublicClientAPIMux, base.SynapseAdminMux, &base.Cfg.ClientAPI, - accountDB, federation, rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, userAPI, + federation, rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, userAPI, keyAPI, nil, &cfg.MSCs, ) diff --git a/setup/monolith.go b/setup/monolith.go index fa6d962c4..8e6240d37 100644 --- a/setup/monolith.go +++ b/setup/monolith.go @@ -57,7 +57,7 @@ type Monolith struct { // AddAllPublicRoutes attaches all public paths to the given router func (m *Monolith) AddAllPublicRoutes(process *process.ProcessContext, csMux, ssMux, keyMux, wkMux, mediaMux, synapseMux *mux.Router) { clientapi.AddPublicRoutes( - process, csMux, synapseMux, &m.Config.ClientAPI, m.AccountDB, + process, csMux, synapseMux, &m.Config.ClientAPI, m.FedClient, m.RoomserverAPI, m.EDUInternalAPI, m.AppserviceAPI, transactions.New(), m.FederationAPI, m.UserAPI, m.KeyAPI, diff --git a/userapi/api/api.go b/userapi/api/api.go index e9cdbe01c..513a060c4 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -27,16 +27,16 @@ import ( // UserInternalAPI is the internal API for information about users and devices. type UserInternalAPI interface { LoginTokenInternalAPI + UserProfileAPI + UserRegisterAPI + UserAccountAPI + UserThreePIDAPI InputAccountData(ctx context.Context, req *InputAccountDataRequest, res *InputAccountDataResponse) error - PerformAccountCreation(ctx context.Context, req *PerformAccountCreationRequest, res *PerformAccountCreationResponse) error - PerformPasswordUpdate(ctx context.Context, req *PerformPasswordUpdateRequest, res *PerformPasswordUpdateResponse) error - PerformDeviceCreation(ctx context.Context, req *PerformDeviceCreationRequest, res *PerformDeviceCreationResponse) error PerformDeviceDeletion(ctx context.Context, req *PerformDeviceDeletionRequest, res *PerformDeviceDeletionResponse) error PerformLastSeenUpdate(ctx context.Context, req *PerformLastSeenUpdateRequest, res *PerformLastSeenUpdateResponse) error PerformDeviceUpdate(ctx context.Context, req *PerformDeviceUpdateRequest, res *PerformDeviceUpdateResponse) error - PerformAccountDeactivation(ctx context.Context, req *PerformAccountDeactivationRequest, res *PerformAccountDeactivationResponse) error PerformOpenIDTokenCreation(ctx context.Context, req *PerformOpenIDTokenCreationRequest, res *PerformOpenIDTokenCreationResponse) error PerformKeyBackup(ctx context.Context, req *PerformKeyBackupRequest, res *PerformKeyBackupResponse) error PerformPusherSet(ctx context.Context, req *PerformPusherSetRequest, res *struct{}) error @@ -44,18 +44,47 @@ type UserInternalAPI interface { PerformPushRulesPut(ctx context.Context, req *PerformPushRulesPutRequest, res *struct{}) error QueryKeyBackup(ctx context.Context, req *QueryKeyBackupRequest, res *QueryKeyBackupResponse) - QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error QueryAccessToken(ctx context.Context, req *QueryAccessTokenRequest, res *QueryAccessTokenResponse) error QueryDevices(ctx context.Context, req *QueryDevicesRequest, res *QueryDevicesResponse) error QueryAccountData(ctx context.Context, req *QueryAccountDataRequest, res *QueryAccountDataResponse) error QueryDeviceInfos(ctx context.Context, req *QueryDeviceInfosRequest, res *QueryDeviceInfosResponse) error - QuerySearchProfiles(ctx context.Context, req *QuerySearchProfilesRequest, res *QuerySearchProfilesResponse) error QueryOpenIDToken(ctx context.Context, req *QueryOpenIDTokenRequest, res *QueryOpenIDTokenResponse) error QueryPushers(ctx context.Context, req *QueryPushersRequest, res *QueryPushersResponse) error QueryPushRules(ctx context.Context, req *QueryPushRulesRequest, res *QueryPushRulesResponse) error QueryNotifications(ctx context.Context, req *QueryNotificationsRequest, res *QueryNotificationsResponse) error } +// UserProfileAPI provides functions for getting user profiles +type UserProfileAPI interface { + QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error + QuerySearchProfiles(ctx context.Context, req *QuerySearchProfilesRequest, res *QuerySearchProfilesResponse) error + SetAvatarURL(ctx context.Context, req *PerformSetAvatarURLRequest, res *PerformSetAvatarURLResponse) error + SetDisplayName(ctx context.Context, req *PerformUpdateDisplayNameRequest, res *struct{}) error +} + +// UserRegisterAPI defines functions for registering accounts +type UserRegisterAPI interface { + QueryNumericLocalpart(ctx context.Context, res *QueryNumericLocalpartResponse) error + QueryAccountAvailability(ctx context.Context, req *QueryAccountAvailabilityRequest, res *QueryAccountAvailabilityResponse) error + PerformAccountCreation(ctx context.Context, req *PerformAccountCreationRequest, res *PerformAccountCreationResponse) error + PerformDeviceCreation(ctx context.Context, req *PerformDeviceCreationRequest, res *PerformDeviceCreationResponse) error +} + +// UserAccountAPI defines functions for changing an account +type UserAccountAPI interface { + PerformPasswordUpdate(ctx context.Context, req *PerformPasswordUpdateRequest, res *PerformPasswordUpdateResponse) error + PerformAccountDeactivation(ctx context.Context, req *PerformAccountDeactivationRequest, res *PerformAccountDeactivationResponse) error + QueryAccountByPassword(ctx context.Context, req *QueryAccountByPasswordRequest, res *QueryAccountByPasswordResponse) error +} + +// UserThreePIDAPI defines functions for 3PID +type UserThreePIDAPI interface { + QueryLocalpartForThreePID(ctx context.Context, req *QueryLocalpartForThreePIDRequest, res *QueryLocalpartForThreePIDResponse) error + QueryThreePIDsForLocalpart(ctx context.Context, req *QueryThreePIDsForLocalpartRequest, res *QueryThreePIDsForLocalpartResponse) error + PerformForgetThreePID(ctx context.Context, req *PerformForgetThreePIDRequest, res *struct{}) error + PerformSaveThreePIDAssociation(ctx context.Context, req *PerformSaveThreePIDAssociationRequest, res *struct{}) error +} + type PerformKeyBackupRequest struct { UserID string Version string // optional if modifying a key backup @@ -507,3 +536,55 @@ type Notification struct { RoomID string `json:"room_id"` // Required. TS gomatrixserverlib.Timestamp `json:"ts"` // Required. } + +type PerformSetAvatarURLRequest struct { + Localpart, AvatarURL string +} +type PerformSetAvatarURLResponse struct{} + +type QueryNumericLocalpartResponse struct { + ID int64 +} + +type QueryAccountAvailabilityRequest struct { + Localpart string +} + +type QueryAccountAvailabilityResponse struct { + Available bool +} + +type QueryAccountByPasswordRequest struct { + Localpart, PlaintextPassword string +} + +type QueryAccountByPasswordResponse struct { + Account *Account + Exists bool +} + +type PerformUpdateDisplayNameRequest struct { + Localpart, DisplayName string +} + +type QueryLocalpartForThreePIDRequest struct { + ThreePID, Medium string +} + +type QueryLocalpartForThreePIDResponse struct { + Localpart string +} + +type QueryThreePIDsForLocalpartRequest struct { + Localpart string +} + +type QueryThreePIDsForLocalpartResponse struct { + ThreePIDs []authtypes.ThreePID +} + +type PerformForgetThreePIDRequest QueryLocalpartForThreePIDRequest + +type PerformSaveThreePIDAssociationRequest struct { + ThreePID, Localpart, Medium string +} diff --git a/userapi/api/api_trace.go b/userapi/api/api_trace.go index 9334f4455..6d8d28007 100644 --- a/userapi/api/api_trace.go +++ b/userapi/api/api_trace.go @@ -149,6 +149,60 @@ func (t *UserInternalAPITrace) QueryNotifications(ctx context.Context, req *Quer return err } +func (t *UserInternalAPITrace) SetAvatarURL(ctx context.Context, req *PerformSetAvatarURLRequest, res *PerformSetAvatarURLResponse) error { + err := t.Impl.SetAvatarURL(ctx, req, res) + util.GetLogger(ctx).Infof("SetAvatarURL req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *UserInternalAPITrace) QueryNumericLocalpart(ctx context.Context, res *QueryNumericLocalpartResponse) error { + err := t.Impl.QueryNumericLocalpart(ctx, res) + util.GetLogger(ctx).Infof("QueryNumericLocalpart req= res=%+v", js(res)) + return err +} + +func (t *UserInternalAPITrace) QueryAccountAvailability(ctx context.Context, req *QueryAccountAvailabilityRequest, res *QueryAccountAvailabilityResponse) error { + err := t.Impl.QueryAccountAvailability(ctx, req, res) + util.GetLogger(ctx).Infof("QueryAccountAvailability req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *UserInternalAPITrace) SetDisplayName(ctx context.Context, req *PerformUpdateDisplayNameRequest, res *struct{}) error { + err := t.Impl.SetDisplayName(ctx, req, res) + util.GetLogger(ctx).Infof("SetDisplayName req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *UserInternalAPITrace) QueryAccountByPassword(ctx context.Context, req *QueryAccountByPasswordRequest, res *QueryAccountByPasswordResponse) error { + err := t.Impl.QueryAccountByPassword(ctx, req, res) + util.GetLogger(ctx).Infof("QueryAccountByPassword req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *UserInternalAPITrace) QueryLocalpartForThreePID(ctx context.Context, req *QueryLocalpartForThreePIDRequest, res *QueryLocalpartForThreePIDResponse) error { + err := t.Impl.QueryLocalpartForThreePID(ctx, req, res) + util.GetLogger(ctx).Infof("QueryLocalpartForThreePID req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *UserInternalAPITrace) QueryThreePIDsForLocalpart(ctx context.Context, req *QueryThreePIDsForLocalpartRequest, res *QueryThreePIDsForLocalpartResponse) error { + err := t.Impl.QueryThreePIDsForLocalpart(ctx, req, res) + util.GetLogger(ctx).Infof("QueryThreePIDsForLocalpart req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *UserInternalAPITrace) PerformForgetThreePID(ctx context.Context, req *PerformForgetThreePIDRequest, res *struct{}) error { + err := t.Impl.PerformForgetThreePID(ctx, req, res) + util.GetLogger(ctx).Infof("PerformForgetThreePID req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *UserInternalAPITrace) PerformSaveThreePIDAssociation(ctx context.Context, req *PerformSaveThreePIDAssociationRequest, res *struct{}) error { + err := t.Impl.PerformSaveThreePIDAssociation(ctx, req, res) + util.GetLogger(ctx).Infof("PerformSaveThreePIDAssociation req=%+v res=%+v", js(req), js(res)) + return err +} + func js(thing interface{}) string { b, err := json.Marshal(thing) if err != nil { diff --git a/userapi/internal/api.go b/userapi/internal/api.go index 7a42fc605..afe57da25 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -26,6 +26,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/sirupsen/logrus" + "golang.org/x/crypto/bcrypt" "github.com/matrix-org/dendrite/appservice/types" "github.com/matrix-org/dendrite/clientapi/userutil" @@ -761,4 +762,71 @@ func (a *UserInternalAPI) QueryPushRules(ctx context.Context, req *api.QueryPush return nil } +func (a *UserInternalAPI) SetAvatarURL(ctx context.Context, req *api.PerformSetAvatarURLRequest, res *api.PerformSetAvatarURLResponse) error { + return a.DB.SetAvatarURL(ctx, req.Localpart, req.AvatarURL) +} + +func (a *UserInternalAPI) QueryNumericLocalpart(ctx context.Context, res *api.QueryNumericLocalpartResponse) error { + id, err := a.DB.GetNewNumericLocalpart(ctx) + if err != nil { + return err + } + res.ID = id + return nil +} + +func (a *UserInternalAPI) QueryAccountAvailability(ctx context.Context, req *api.QueryAccountAvailabilityRequest, res *api.QueryAccountAvailabilityResponse) error { + _, err := a.DB.CheckAccountAvailability(ctx, req.Localpart) + if err == sql.ErrNoRows { + res.Available = true + return nil + } + res.Available = false + return err +} + +func (a *UserInternalAPI) QueryAccountByPassword(ctx context.Context, req *api.QueryAccountByPasswordRequest, res *api.QueryAccountByPasswordResponse) error { + acc, err := a.DB.GetAccountByPassword(ctx, req.Localpart, req.PlaintextPassword) + switch err { + case sql.ErrNoRows: // user does not exist + return nil + case bcrypt.ErrMismatchedHashAndPassword: // user exists, but password doesn't match + return nil + default: + res.Exists = true + res.Account = acc + return nil + } +} + +func (a *UserInternalAPI) SetDisplayName(ctx context.Context, req *api.PerformUpdateDisplayNameRequest, _ *struct{}) error { + return a.DB.SetDisplayName(ctx, req.Localpart, req.DisplayName) +} + +func (a *UserInternalAPI) QueryLocalpartForThreePID(ctx context.Context, req *api.QueryLocalpartForThreePIDRequest, res *api.QueryLocalpartForThreePIDResponse) error { + localpart, err := a.DB.GetLocalpartForThreePID(ctx, req.ThreePID, req.Medium) + if err != nil { + return err + } + res.Localpart = localpart + return nil +} + +func (a *UserInternalAPI) QueryThreePIDsForLocalpart(ctx context.Context, req *api.QueryThreePIDsForLocalpartRequest, res *api.QueryThreePIDsForLocalpartResponse) error { + r, err := a.DB.GetThreePIDsForLocalpart(ctx, req.Localpart) + if err != nil { + return err + } + res.ThreePIDs = r + return nil +} + +func (a *UserInternalAPI) PerformForgetThreePID(ctx context.Context, req *api.PerformForgetThreePIDRequest, res *struct{}) error { + return a.DB.RemoveThreePIDAssociation(ctx, req.ThreePID, req.Medium) +} + +func (a *UserInternalAPI) PerformSaveThreePIDAssociation(ctx context.Context, req *api.PerformSaveThreePIDAssociationRequest, res *struct{}) error { + return a.DB.SaveThreePIDAssociation(ctx, req.ThreePID, req.Localpart, req.Medium) +} + const pushRulesAccountDataType = "m.push_rules" diff --git a/userapi/inthttp/client.go b/userapi/inthttp/client.go index 8ec649ad0..23c335cf2 100644 --- a/userapi/inthttp/client.go +++ b/userapi/inthttp/client.go @@ -28,30 +28,39 @@ import ( const ( InputAccountDataPath = "/userapi/inputAccountData" - PerformDeviceCreationPath = "/userapi/performDeviceCreation" - PerformAccountCreationPath = "/userapi/performAccountCreation" - PerformPasswordUpdatePath = "/userapi/performPasswordUpdate" - PerformDeviceDeletionPath = "/userapi/performDeviceDeletion" - PerformLastSeenUpdatePath = "/userapi/performLastSeenUpdate" - PerformDeviceUpdatePath = "/userapi/performDeviceUpdate" - PerformAccountDeactivationPath = "/userapi/performAccountDeactivation" - PerformOpenIDTokenCreationPath = "/userapi/performOpenIDTokenCreation" - PerformKeyBackupPath = "/userapi/performKeyBackup" - PerformPusherSetPath = "/pushserver/performPusherSet" - PerformPusherDeletionPath = "/pushserver/performPusherDeletion" - PerformPushRulesPutPath = "/pushserver/performPushRulesPut" + PerformDeviceCreationPath = "/userapi/performDeviceCreation" + PerformAccountCreationPath = "/userapi/performAccountCreation" + PerformPasswordUpdatePath = "/userapi/performPasswordUpdate" + PerformDeviceDeletionPath = "/userapi/performDeviceDeletion" + PerformLastSeenUpdatePath = "/userapi/performLastSeenUpdate" + PerformDeviceUpdatePath = "/userapi/performDeviceUpdate" + PerformAccountDeactivationPath = "/userapi/performAccountDeactivation" + PerformOpenIDTokenCreationPath = "/userapi/performOpenIDTokenCreation" + PerformKeyBackupPath = "/userapi/performKeyBackup" + PerformPusherSetPath = "/pushserver/performPusherSet" + PerformPusherDeletionPath = "/pushserver/performPusherDeletion" + PerformPushRulesPutPath = "/pushserver/performPushRulesPut" + PerformSetAvatarURLPath = "/userapi/performSetAvatarURL" + PerformSetDisplayNamePath = "/userapi/performSetDisplayName" + PerformForgetThreePIDPath = "/userapi/performForgetThreePID" + PerformSaveThreePIDAssociationPath = "/userapi/performSaveThreePIDAssociation" - QueryKeyBackupPath = "/userapi/queryKeyBackup" - QueryProfilePath = "/userapi/queryProfile" - QueryAccessTokenPath = "/userapi/queryAccessToken" - QueryDevicesPath = "/userapi/queryDevices" - QueryAccountDataPath = "/userapi/queryAccountData" - QueryDeviceInfosPath = "/userapi/queryDeviceInfos" - QuerySearchProfilesPath = "/userapi/querySearchProfiles" - QueryOpenIDTokenPath = "/userapi/queryOpenIDToken" - QueryPushersPath = "/pushserver/queryPushers" - QueryPushRulesPath = "/pushserver/queryPushRules" - QueryNotificationsPath = "/pushserver/queryNotifications" + QueryKeyBackupPath = "/userapi/queryKeyBackup" + QueryProfilePath = "/userapi/queryProfile" + QueryAccessTokenPath = "/userapi/queryAccessToken" + QueryDevicesPath = "/userapi/queryDevices" + QueryAccountDataPath = "/userapi/queryAccountData" + QueryDeviceInfosPath = "/userapi/queryDeviceInfos" + QuerySearchProfilesPath = "/userapi/querySearchProfiles" + QueryOpenIDTokenPath = "/userapi/queryOpenIDToken" + QueryPushersPath = "/pushserver/queryPushers" + QueryPushRulesPath = "/pushserver/queryPushRules" + QueryNotificationsPath = "/pushserver/queryNotifications" + QueryNumericLocalpartPath = "/userapi/queryNumericLocalpart" + QueryAccountAvailabilityPath = "/userapi/queryAccountAvailability" + QueryAccountByPasswordPath = "/userapi/queryAccountByPassword" + QueryLocalpartForThreePIDPath = "/userapi/queryLocalpartForThreePID" + QueryThreePIDsForLocalpartPath = "/userapi/queryThreePIDsForLocalpart" ) // NewUserAPIClient creates a UserInternalAPI implemented by talking to a HTTP POST API. @@ -310,3 +319,75 @@ func (h *httpUserInternalAPI) QueryPushRules(ctx context.Context, req *api.Query apiURL := h.apiURL + QueryPushRulesPath return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res) } + +func (h *httpUserInternalAPI) SetAvatarURL(ctx context.Context, req *api.PerformSetAvatarURLRequest, res *api.PerformSetAvatarURLResponse) error { + span, ctx := opentracing.StartSpanFromContext(ctx, PerformSetAvatarURLPath) + defer span.Finish() + + apiURL := h.apiURL + PerformSetAvatarURLPath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res) +} + +func (h *httpUserInternalAPI) QueryNumericLocalpart(ctx context.Context, res *api.QueryNumericLocalpartResponse) error { + span, ctx := opentracing.StartSpanFromContext(ctx, QueryNumericLocalpartPath) + defer span.Finish() + + apiURL := h.apiURL + QueryNumericLocalpartPath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, struct{}{}, res) +} + +func (h *httpUserInternalAPI) QueryAccountAvailability(ctx context.Context, req *api.QueryAccountAvailabilityRequest, res *api.QueryAccountAvailabilityResponse) error { + span, ctx := opentracing.StartSpanFromContext(ctx, QueryAccountAvailabilityPath) + defer span.Finish() + + apiURL := h.apiURL + QueryAccountAvailabilityPath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res) +} + +func (h *httpUserInternalAPI) QueryAccountByPassword(ctx context.Context, req *api.QueryAccountByPasswordRequest, res *api.QueryAccountByPasswordResponse) error { + span, ctx := opentracing.StartSpanFromContext(ctx, QueryAccountByPasswordPath) + defer span.Finish() + + apiURL := h.apiURL + QueryAccountByPasswordPath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res) +} + +func (h *httpUserInternalAPI) SetDisplayName(ctx context.Context, req *api.PerformUpdateDisplayNameRequest, res *struct{}) error { + span, ctx := opentracing.StartSpanFromContext(ctx, PerformSetDisplayNamePath) + defer span.Finish() + + apiURL := h.apiURL + PerformSetDisplayNamePath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res) +} + +func (h *httpUserInternalAPI) QueryLocalpartForThreePID(ctx context.Context, req *api.QueryLocalpartForThreePIDRequest, res *api.QueryLocalpartForThreePIDResponse) error { + span, ctx := opentracing.StartSpanFromContext(ctx, QueryLocalpartForThreePIDPath) + defer span.Finish() + + apiURL := h.apiURL + QueryLocalpartForThreePIDPath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res) +} + +func (h *httpUserInternalAPI) QueryThreePIDsForLocalpart(ctx context.Context, req *api.QueryThreePIDsForLocalpartRequest, res *api.QueryThreePIDsForLocalpartResponse) error { + span, ctx := opentracing.StartSpanFromContext(ctx, QueryThreePIDsForLocalpartPath) + defer span.Finish() + + apiURL := h.apiURL + QueryThreePIDsForLocalpartPath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res) +} + +func (h *httpUserInternalAPI) PerformForgetThreePID(ctx context.Context, req *api.PerformForgetThreePIDRequest, res *struct{}) error { + span, ctx := opentracing.StartSpanFromContext(ctx, PerformForgetThreePIDPath) + defer span.Finish() + + apiURL := h.apiURL + PerformForgetThreePIDPath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res) +} + +func (h *httpUserInternalAPI) PerformSaveThreePIDAssociation(ctx context.Context, req *api.PerformSaveThreePIDAssociationRequest, res *struct{}) error { + span, ctx := opentracing.StartSpanFromContext(ctx, PerformSaveThreePIDAssociationPath) + defer span.Finish() + + apiURL := h.apiURL + PerformSaveThreePIDAssociationPath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res) +} diff --git a/userapi/inthttp/server.go b/userapi/inthttp/server.go index 526f99575..f229aa3bb 100644 --- a/userapi/inthttp/server.go +++ b/userapi/inthttp/server.go @@ -347,4 +347,101 @@ func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + internalAPIMux.Handle(PerformSetAvatarURLPath, + httputil.MakeInternalAPI("performSetAvatarURL", func(req *http.Request) util.JSONResponse { + request := api.PerformSetAvatarURLRequest{} + response := api.PerformSetAvatarURLResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := s.SetAvatarURL(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle(QueryNumericLocalpartPath, + httputil.MakeInternalAPI("queryNumericLocalpart", func(req *http.Request) util.JSONResponse { + response := api.QueryNumericLocalpartResponse{} + if err := s.QueryNumericLocalpart(req.Context(), &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle(QueryAccountByPasswordPath, + httputil.MakeInternalAPI("queryAccountByPassword", func(req *http.Request) util.JSONResponse { + request := api.QueryAccountByPasswordRequest{} + response := api.QueryAccountByPasswordResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := s.QueryAccountByPassword(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle(PerformSetDisplayNamePath, + httputil.MakeInternalAPI("performSetDisplayName", func(req *http.Request) util.JSONResponse { + request := api.PerformUpdateDisplayNameRequest{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := s.SetDisplayName(req.Context(), &request, &struct{}{}); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &struct{}{}} + }), + ) + internalAPIMux.Handle(QueryLocalpartForThreePIDPath, + httputil.MakeInternalAPI("queryLocalpartForThreePID", func(req *http.Request) util.JSONResponse { + request := api.QueryLocalpartForThreePIDRequest{} + response := api.QueryLocalpartForThreePIDResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := s.QueryLocalpartForThreePID(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle(QueryThreePIDsForLocalpartPath, + httputil.MakeInternalAPI("queryThreePIDsForLocalpart", func(req *http.Request) util.JSONResponse { + request := api.QueryThreePIDsForLocalpartRequest{} + response := api.QueryThreePIDsForLocalpartResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := s.QueryThreePIDsForLocalpart(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle(PerformForgetThreePIDPath, + httputil.MakeInternalAPI("performForgetThreePID", func(req *http.Request) util.JSONResponse { + request := api.PerformForgetThreePIDRequest{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := s.PerformForgetThreePID(req.Context(), &request, &struct{}{}); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &struct{}{}} + }), + ) + internalAPIMux.Handle(PerformSaveThreePIDAssociationPath, + httputil.MakeInternalAPI("performSaveThreePIDAssociation", func(req *http.Request) util.JSONResponse { + request := api.PerformSaveThreePIDAssociationRequest{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := s.PerformSaveThreePIDAssociation(req.Context(), &request, &struct{}{}); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &struct{}{}} + }), + ) } diff --git a/userapi/storage/interface.go b/userapi/storage/interface.go index 777067109..b15470dd4 100644 --- a/userapi/storage/interface.go +++ b/userapi/storage/interface.go @@ -24,12 +24,17 @@ import ( "github.com/matrix-org/dendrite/userapi/storage/tables" ) -type Database interface { - GetAccountByPassword(ctx context.Context, localpart, plaintextPassword string) (*api.Account, error) +type Profile interface { GetProfileByLocalpart(ctx context.Context, localpart string) (*authtypes.Profile, error) + SearchProfiles(ctx context.Context, searchString string, limit int) ([]authtypes.Profile, error) SetPassword(ctx context.Context, localpart string, plaintextPassword string) error SetAvatarURL(ctx context.Context, localpart string, avatarURL string) error SetDisplayName(ctx context.Context, localpart string, displayName string) error +} + +type Database interface { + Profile + GetAccountByPassword(ctx context.Context, localpart, plaintextPassword string) (*api.Account, error) // CreateAccount makes a new account with the given login name and password, and creates an empty profile // for this account. If no password is supplied, the account will be a passwordless account. If the // account already exists, it will return nil, ErrUserExists. @@ -48,7 +53,6 @@ type Database interface { GetThreePIDsForLocalpart(ctx context.Context, localpart string) (threepids []authtypes.ThreePID, err error) CheckAccountAvailability(ctx context.Context, localpart string) (bool, error) GetAccountByLocalpart(ctx context.Context, localpart string) (*api.Account, error) - SearchProfiles(ctx context.Context, searchString string, limit int) ([]authtypes.Profile, error) DeactivateAccount(ctx context.Context, localpart string) (err error) CreateOpenIDToken(ctx context.Context, token, localpart string) (exp int64, err error) GetOpenIDTokenAttributes(ctx context.Context, token string) (*api.OpenIDTokenAttributes, error) From 87298985a7f94724ab467ab9814bd6c41fcb3027 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 25 Mar 2022 09:05:06 +0000 Subject: [PATCH 19/55] Update Docker workflow some more --- .github/workflows/docker.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2c5d9e5e6..5c59a02a6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -5,10 +5,10 @@ name: "Docker" on: release: # A GitHub release was published types: [published] - workflow_run: # The Dendrite pipeline completed successfully on main - workflows: [Dendrite] - types: [completed] - branches: [main] + #workflow_run: # The Dendrite pipeline completed successfully on main + # workflows: [Dendrite] + # types: [completed] + # branches: [main] workflow_dispatch: # A build was manually requested env: @@ -21,6 +21,9 @@ jobs: monolith: name: Monolith image runs-on: ubuntu-latest + permissions: + contents: read + packages: write steps: - name: Checkout uses: actions/checkout@v2 @@ -40,7 +43,7 @@ jobs: uses: docker/login-action@v1 with: registry: ghcr.io - username: ${{ github.actor }} + username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build monolith image @@ -80,6 +83,9 @@ jobs: polylith: name: Polylith image runs-on: ubuntu-latest + permissions: + contents: read + packages: write steps: - name: Checkout uses: actions/checkout@v2 @@ -99,7 +105,7 @@ jobs: uses: docker/login-action@v1 with: registry: ghcr.io - username: ${{ github.actor }} + username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build polylith image From 5e780d3ca232af08079c7bebb6166519dda1906c Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 25 Mar 2022 10:08:13 +0000 Subject: [PATCH 20/55] Chain Docker update onto `main` --- .github/workflows/dendrite.yml | 33 +++++++++++++++++++++++---------- .github/workflows/docker.yml | 17 +++++------------ 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index d337865f2..715235d3d 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -76,7 +76,7 @@ jobs: strategy: fail-fast: false matrix: - go: [ '1.16', '1.17', '1.18' ] + go: ["1.16", "1.17", "1.18"] steps: - uses: actions/checkout@v3 - name: Setup go @@ -101,9 +101,9 @@ jobs: strategy: fail-fast: false matrix: - go: [ '1.16', '1.17', '1.18' ] - goos: [ 'linux' ] - goarch: [ 'amd64', '386' ] + go: ["1.16", "1.17", "1.18"] + goos: ["linux"] + goarch: ["amd64", "386"] steps: - uses: actions/checkout@v3 - name: Setup go @@ -134,9 +134,9 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go: [ '1.16', '1.17', '1.18' ] - goos: [ 'windows' ] - goarch: [ 'amd64' ] + go: ["1.16", "1.17", "1.18"] + goos: ["windows"] + goarch: ["amd64"] steps: - uses: actions/checkout@v3 - name: Setup Go ${{ matrix.go }} @@ -163,7 +163,7 @@ jobs: # Dummy step to gate other tests on without repeating the whole list initial-tests-done: name: Initial tests passed - needs: [ lint, test, build, build_windows ] + needs: [lint, test, build, build_windows] runs-on: ubuntu-latest if: ${{ !cancelled() }} # Run this even if prior jobs were skipped steps: @@ -183,7 +183,7 @@ jobs: - name: Setup go uses: actions/setup-go@v2 with: - go-version: '1.16' + go-version: "1.16" - uses: actions/cache@v3 with: path: | @@ -232,7 +232,7 @@ jobs: working-directory: /src - name: Summarise results.tap if: ${{ always() }} - run: /sytest/scripts/tap_to_gha.pl /logs/results.tap + run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - name: Upload Sytest logs uses: actions/upload-artifact@v2 @@ -322,3 +322,16 @@ jobs: COMPLEMENT_BASE_IMAGE: complement-dendrite:latest API: ${{ matrix.api && 1 }} working-directory: complement + + update-docker-images: + name: Update Docker images + if: github.repository == 'matrix-org/dendrite' && github.ref_name == 'main' + needs: [initial-tests-done, upgrade_test, sytest, complement] + runs-on: ubuntu-latest + steps: + - name: Check integration tests passed + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} + - name: Dispatch Docker build + uses: ./.github/workflows/dendrite.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5c59a02a6..129a80487 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -5,11 +5,8 @@ name: "Docker" on: release: # A GitHub release was published types: [published] - #workflow_run: # The Dendrite pipeline completed successfully on main - # workflows: [Dendrite] - # types: [completed] - # branches: [main] workflow_dispatch: # A build was manually requested + workflow_call: # Another pipeline called us env: DOCKER_NAMESPACE: matrixdotorg @@ -46,10 +43,8 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build monolith image - if: >- - (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') || - github.event_name == 'workflow_dispatch' + - name: Build main monolith image + if: github.ref_name == 'main' id: docker_build_monolith uses: docker/build-push-action@v2 with: @@ -108,10 +103,8 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build polylith image - if: >- - (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') || - github.event_name == 'workflow_dispatch' + - name: Build main polylith image + if: github.ref_name == 'main' id: docker_build_polylith uses: docker/build-push-action@v2 with: From e6d4bdeed5a05f26677f81c02f7a43c84a4a947e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 25 Mar 2022 12:24:21 +0000 Subject: [PATCH 21/55] Try to recover from corrupted NATS streams in memory temporarily (#2301) --- setup/jetstream/nats.go | 43 ++++++++++++++++++++++++++++++++++++---- setup/process/process.go | 17 ++++++++++++++++ 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/setup/jetstream/nats.go b/setup/jetstream/nats.go index 748c191b0..328cf9155 100644 --- a/setup/jetstream/nats.go +++ b/setup/jetstream/nats.go @@ -1,11 +1,13 @@ package jetstream import ( + "fmt" "reflect" "strings" "sync" "time" + "github.com/getsentry/sentry-go" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/process" "github.com/sirupsen/logrus" @@ -20,7 +22,7 @@ var natsServerMutex sync.Mutex func Prepare(process *process.ProcessContext, cfg *config.JetStream) (natsclient.JetStreamContext, *natsclient.Conn) { // check if we need an in-process NATS Server if len(cfg.Addresses) != 0 { - return setupNATS(cfg, nil) + return setupNATS(process, cfg, nil) } natsServerMutex.Lock() if natsServer == nil { @@ -56,10 +58,10 @@ func Prepare(process *process.ProcessContext, cfg *config.JetStream) (natsclient if err != nil { logrus.Fatalln("Failed to create NATS client") } - return setupNATS(cfg, nc) + return setupNATS(process, cfg, nc) } -func setupNATS(cfg *config.JetStream, nc *natsclient.Conn) (natsclient.JetStreamContext, *natsclient.Conn) { +func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsclient.Conn) (natsclient.JetStreamContext, *natsclient.Conn) { if nc == nil { var err error nc, err = natsclient.Connect(strings.Join(cfg.Addresses, ",")) @@ -117,7 +119,40 @@ func setupNATS(cfg *config.JetStream, nc *natsclient.Conn) (natsclient.JetStream namespaced.Name = name namespaced.Subjects = subjects if _, err = s.AddStream(&namespaced); err != nil { - logrus.WithError(err).WithField("stream", name).WithField("subjects", subjects).Fatal("Unable to add stream") + logger := logrus.WithError(err).WithFields(logrus.Fields{ + "stream": namespaced.Name, + "subjects": namespaced.Subjects, + }) + + // If the stream was supposed to be in-memory to begin with + // then an error here is fatal so we'll give up. + if namespaced.Storage == natsclient.MemoryStorage { + logger.WithError(err).Fatal("Unable to add in-memory stream") + } + + // The stream was supposed to be on disk. Let's try starting + // Dendrite with the stream in-memory instead. That'll mean that + // we can't recover anything that was queued on the disk but we + // will still be able to start and run hopefully in the meantime. + logger.WithError(err).Error("Unable to add stream") + sentry.CaptureException(fmt.Errorf("Unable to add stream %q: %w", namespaced.Name, err)) + + namespaced.Storage = natsclient.MemoryStorage + if _, err = s.AddStream(&namespaced); err != nil { + // We tried to add the stream in-memory instead but something + // went wrong. That's an unrecoverable situation so we will + // give up at this point. + logger.WithError(err).Fatal("Unable to add in-memory stream") + } + + if stream.Storage != namespaced.Storage { + // We've managed to add the stream in memory. What's on the + // disk will be left alone, but our ability to recover from a + // future crash will be limited. Yell about it. + sentry.CaptureException(fmt.Errorf("Stream %q is running in-memory; this may be due to data corruption in the JetStream storage directory, investigate as soon as possible", namespaced.Name)) + logrus.Warn("Stream is running in-memory; this may be due to data corruption in the JetStream storage directory, investigate as soon as possible") + process.Degraded() + } } } } diff --git a/setup/process/process.go b/setup/process/process.go index d55751d77..01eb26e22 100644 --- a/setup/process/process.go +++ b/setup/process/process.go @@ -2,13 +2,19 @@ package process import ( "context" + "fmt" "sync" + + "github.com/getsentry/sentry-go" + "github.com/sirupsen/logrus" + "go.uber.org/atomic" ) type ProcessContext struct { wg *sync.WaitGroup // used to wait for components to shutdown ctx context.Context // cancelled when Stop is called shutdown context.CancelFunc // shut down Dendrite + degraded atomic.Bool } func NewProcessContext() *ProcessContext { @@ -43,3 +49,14 @@ func (b *ProcessContext) WaitForShutdown() <-chan struct{} { func (b *ProcessContext) WaitForComponentsToFinish() { b.wg.Wait() } + +func (b *ProcessContext) Degraded() { + if b.degraded.CAS(false, true) { + logrus.Warn("Dendrite is running in a degraded state") + sentry.CaptureException(fmt.Errorf("Process is running in a degraded state")) + } +} + +func (b *ProcessContext) IsDegraded() bool { + return b.degraded.Load() +} From b113217a6d4023aa2b0c468eb2be38dd493ad821 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 25 Mar 2022 12:38:16 +0000 Subject: [PATCH 22/55] Use most recent event in response to get latest stream position in incremental sync (#2302) * Use latest event position in response for advancing the stream position in an incremental sync * Create some calm * Use To in worst case * Don't waste CPU cycles on an empty response after all * Bug fixes * Fix another bug --- syncapi/streams/stream_pdu.go | 56 ++++++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index 1afcbe750..ccdac0864 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -147,7 +147,6 @@ func (p *PDUStreamProvider) IncrementalSync( To: to, Backwards: from > to, } - newPos = to var err error var stateDeltas []types.StateDelta @@ -172,14 +171,26 @@ func (p *PDUStreamProvider) IncrementalSync( req.Rooms[roomID] = gomatrixserverlib.Join } + if len(stateDeltas) == 0 { + return to + } + + newPos = from for _, delta := range stateDeltas { - if err = p.addRoomDeltaToResponse(ctx, req.Device, r, delta, &eventFilter, req.Response); err != nil { + var pos types.StreamPosition + if pos, err = p.addRoomDeltaToResponse(ctx, req.Device, r, delta, &eventFilter, req.Response); err != nil { req.Log.WithError(err).Error("d.addRoomDeltaToResponse failed") - return newPos + return to + } + switch { + case r.Backwards && pos < newPos: + fallthrough + case !r.Backwards && pos > newPos: + newPos = pos } } - return r.To + return newPos } func (p *PDUStreamProvider) addRoomDeltaToResponse( @@ -189,7 +200,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( delta types.StateDelta, eventFilter *gomatrixserverlib.RoomEventFilter, res *types.Response, -) error { +) (types.StreamPosition, error) { if delta.MembershipPos > 0 && delta.Membership == gomatrixserverlib.Leave { // make sure we don't leak recent events after the leave event. // TODO: History visibility makes this somewhat complex to handle correctly. For example: @@ -204,19 +215,42 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( eventFilter, true, true, ) if err != nil { - return err + return r.From, err } recentEvents := p.DB.StreamEventsToEvents(device, recentStreamEvents) delta.StateEvents = removeDuplicates(delta.StateEvents, recentEvents) // roll back prevBatch, err := p.DB.GetBackwardTopologyPos(ctx, recentStreamEvents) if err != nil { - return err + return r.From, err } - // XXX: should we ever get this far if we have no recent events or state in this room? - // in practice we do for peeks, but possibly not joins? + // If we didn't return any events at all then don't bother doing anything else. if len(recentEvents) == 0 && len(delta.StateEvents) == 0 { - return nil + return r.To, nil + } + + // Sort the events so that we can pick out the latest events from both sections. + recentEvents = gomatrixserverlib.HeaderedReverseTopologicalOrdering(recentEvents, gomatrixserverlib.TopologicalOrderByPrevEvents) + delta.StateEvents = gomatrixserverlib.HeaderedReverseTopologicalOrdering(delta.StateEvents, gomatrixserverlib.TopologicalOrderByAuthEvents) + + // Work out what the highest stream position is for all of the events in this + // room that were returned. + latestPosition := r.To + updateLatestPosition := func(mostRecentEventID string) { + if _, pos, err := p.DB.PositionInTopology(ctx, mostRecentEventID); err == nil { + switch { + case r.Backwards && pos > latestPosition: + fallthrough + case !r.Backwards && pos < latestPosition: + latestPosition = pos + } + } + } + if len(recentEvents) > 0 { + updateLatestPosition(recentEvents[len(recentEvents)-1].EventID()) + } + if len(delta.StateEvents) > 0 { + updateLatestPosition(delta.StateEvents[len(delta.StateEvents)-1].EventID()) } switch delta.Membership { @@ -250,7 +284,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( res.Rooms.Leave[delta.RoomID] = *lr } - return nil + return latestPosition, nil } func (p *PDUStreamProvider) getJoinResponseForCompleteSync( From 28642683fcc0e36ff7d955db3cdc46b1d65a248b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 25 Mar 2022 13:25:02 +0000 Subject: [PATCH 23/55] Fix Docker flow --- .github/workflows/dendrite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index 715235d3d..61285bf80 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -334,4 +334,4 @@ jobs: with: jobs: ${{ toJSON(needs) }} - name: Dispatch Docker build - uses: ./.github/workflows/dendrite.yml + uses: matrix-org/dendrite/.github/workflows/dendrite.yml From f25afa1e197d8f91f12872687c2be748d955bdf1 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 25 Mar 2022 13:25:15 +0000 Subject: [PATCH 24/55] Healthcheck endpoints (#2303) * Health monitoring endpoints * Rename endpoints --- internal/httputil/paths.go | 2 ++ setup/base/base.go | 18 ++++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/internal/httputil/paths.go b/internal/httputil/paths.go index a1009fc2e..12cf59eb4 100644 --- a/internal/httputil/paths.go +++ b/internal/httputil/paths.go @@ -21,4 +21,6 @@ const ( PublicMediaPathPrefix = "/_matrix/media/" PublicWellKnownPrefix = "/.well-known/matrix/" InternalPathPrefix = "/api/" + DendriteAdminPathPrefix = "/_dendrite/" + SynapseAdminPathPrefix = "/_synapse/" ) diff --git a/setup/base/base.go b/setup/base/base.go index 692a77d5c..c3ed05416 100644 --- a/setup/base/base.go +++ b/setup/base/base.go @@ -76,6 +76,7 @@ type BaseDendrite struct { PublicMediaAPIMux *mux.Router PublicWellKnownAPIMux *mux.Router InternalAPIMux *mux.Router + DendriteAdminMux *mux.Router SynapseAdminMux *mux.Router UseHTTPAPIs bool apiHttpClient *http.Client @@ -208,7 +209,8 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base PublicMediaAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicMediaPathPrefix).Subrouter().UseEncodedPath(), PublicWellKnownAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicWellKnownPrefix).Subrouter().UseEncodedPath(), InternalAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.InternalPathPrefix).Subrouter().UseEncodedPath(), - SynapseAdminMux: mux.NewRouter().SkipClean(true).PathPrefix("/_synapse/").Subrouter().UseEncodedPath(), + DendriteAdminMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.DendriteAdminPathPrefix).Subrouter().UseEncodedPath(), + SynapseAdminMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.SynapseAdminPathPrefix).Subrouter().UseEncodedPath(), apiHttpClient: &apiClient, } } @@ -377,6 +379,17 @@ func (b *BaseDendrite) SetupAndServeHTTP( internalRouter.Handle("/metrics", httputil.WrapHandlerInBasicAuth(promhttp.Handler(), b.Cfg.Global.Metrics.BasicAuth)) } + b.DendriteAdminMux.HandleFunc("/monitor/up", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + }) + b.DendriteAdminMux.HandleFunc("/monitor/health", func(w http.ResponseWriter, r *http.Request) { + if b.ProcessContext.IsDegraded() { + w.WriteHeader(503) + return + } + w.WriteHeader(200) + }) + var clientHandler http.Handler clientHandler = b.PublicClientAPIMux if b.Cfg.Global.Sentry.Enabled { @@ -393,12 +406,13 @@ func (b *BaseDendrite) SetupAndServeHTTP( }) federationHandler = sentryHandler.Handle(b.PublicFederationAPIMux) } + internalRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(b.DendriteAdminMux) externalRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(clientHandler) if !b.Cfg.Global.DisableFederation { externalRouter.PathPrefix(httputil.PublicKeyPathPrefix).Handler(b.PublicKeyAPIMux) externalRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(federationHandler) } - externalRouter.PathPrefix("/_synapse/").Handler(b.SynapseAdminMux) + externalRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(b.SynapseAdminMux) externalRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(b.PublicMediaAPIMux) externalRouter.PathPrefix(httputil.PublicWellKnownPrefix).Handler(b.PublicWellKnownAPIMux) From c8e1ad59976807f4a673508a3d9bc5dc1ec5e7ce Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 25 Mar 2022 13:26:18 +0000 Subject: [PATCH 25/55] Specify branch name in Docker flow --- .github/workflows/dendrite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index 61285bf80..e76e114db 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -334,4 +334,4 @@ jobs: with: jobs: ${{ toJSON(needs) }} - name: Dispatch Docker build - uses: matrix-org/dendrite/.github/workflows/dendrite.yml + uses: matrix-org/dendrite/.github/workflows/dendrite.yml@main From 62bd5592751e920bf0b1d04bc5e5ad28dc7a5c33 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 25 Mar 2022 13:28:20 +0000 Subject: [PATCH 26/55] Factor Docker step into own job --- .github/workflows/dendrite.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index e76e114db..8c4fc2259 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -323,9 +323,8 @@ jobs: API: ${{ matrix.api && 1 }} working-directory: complement - update-docker-images: - name: Update Docker images - if: github.repository == 'matrix-org/dendrite' && github.ref_name == 'main' + integration-tests-done: + name: Integration tests passed needs: [initial-tests-done, upgrade_test, sytest, complement] runs-on: ubuntu-latest steps: @@ -333,5 +332,9 @@ jobs: uses: re-actors/alls-green@release/v1 with: jobs: ${{ toJSON(needs) }} - - name: Dispatch Docker build - uses: matrix-org/dendrite/.github/workflows/dendrite.yml@main + + update-docker-images: + name: Update Docker images + if: github.repository == 'matrix-org/dendrite' && github.ref_name == 'main' + needs: [integration-tests-done] + uses: matrix-org/dendrite/.github/workflows/dendrite.yml@main From 7f3d42bb467f9eb65569d743b818af0e1faff329 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 25 Mar 2022 13:29:09 +0000 Subject: [PATCH 27/55] Use correct path name --- .github/workflows/dendrite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index 8c4fc2259..82055199f 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -337,4 +337,4 @@ jobs: name: Update Docker images if: github.repository == 'matrix-org/dendrite' && github.ref_name == 'main' needs: [integration-tests-done] - uses: matrix-org/dendrite/.github/workflows/dendrite.yml@main + uses: matrix-org/dendrite/.github/workflows/docker.yml@main From 5b5e6a59b65a50453cb467b48f15d52a21f0b9a7 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 25 Mar 2022 13:31:41 +0000 Subject: [PATCH 28/55] Give packages permission to `update-docker-images` --- .github/workflows/dendrite.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index 82055199f..e1f71ea2f 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -335,6 +335,9 @@ jobs: update-docker-images: name: Update Docker images + permissions: + packages: write + contents: read if: github.repository == 'matrix-org/dendrite' && github.ref_name == 'main' needs: [integration-tests-done] uses: matrix-org/dendrite/.github/workflows/docker.yml@main From 873c4d7e2c4256444d03700446c6a814fff6f958 Mon Sep 17 00:00:00 2001 From: S7evinK <2353100+S7evinK@users.noreply.github.com> Date: Fri, 25 Mar 2022 14:38:24 +0100 Subject: [PATCH 29/55] Fixes for `create-account` (#2285) * Check user existence Fallback to asking for the password if non is defined * Add missing tests * Update to not use pointers, verify username length * Re-add possibilty to create passwordless account * Fix config issue * Fix test again Co-authored-by: Neil Alexander --- cmd/create-account/main.go | 81 +++++++++++++++++++-------------- cmd/create-account/main_test.go | 41 ++++++++++------- 2 files changed, 72 insertions(+), 50 deletions(-) diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go index 4d01a9f42..2719f8680 100644 --- a/cmd/create-account/main.go +++ b/cmd/create-account/main.go @@ -24,12 +24,11 @@ import ( "regexp" "strings" + "github.com/matrix-org/dendrite/setup" "github.com/matrix-org/dendrite/setup/base" + "github.com/matrix-org/dendrite/userapi/api" "github.com/sirupsen/logrus" "golang.org/x/term" - - "github.com/matrix-org/dendrite/setup" - "github.com/matrix-org/dendrite/userapi/api" ) const usage = `Usage: %s @@ -43,7 +42,7 @@ Example: # use password from file %s --config dendrite.yaml -username alice -passwordfile my.pass # ask user to provide password - %s --config dendrite.yaml -username alice -ask-pass + %s --config dendrite.yaml -username alice # read password from stdin %s --config dendrite.yaml -username alice -passwordstdin < my.pass cat my.pass | %s --config dendrite.yaml -username alice -passwordstdin @@ -56,10 +55,10 @@ Arguments: var ( username = flag.String("username", "", "The username of the account to register (specify the localpart only, e.g. 'alice' for '@alice:domain.com')") - password = flag.String("password", "", "The password to associate with the account (optional, account will be password-less if not specified)") + password = flag.String("password", "", "The password to associate with the account") pwdFile = flag.String("passwordfile", "", "The file to use for the password (e.g. for automated account creation)") pwdStdin = flag.Bool("passwordstdin", false, "Reads the password from stdin") - askPass = flag.Bool("ask-pass", false, "Ask for the password to use") + pwdLess = flag.Bool("passwordless", false, "Create a passwordless account, e.g. if only an accesstoken is required") isAdmin = flag.Bool("admin", false, "Create an admin account") resetPassword = flag.Bool("reset-password", false, "Resets the password for the given username") validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-=./]+$`) @@ -78,22 +77,44 @@ func main() { os.Exit(1) } + if *pwdLess && *resetPassword { + logrus.Fatalf("Can not reset to an empty password, unable to login afterwards.") + } + if !validUsernameRegex.MatchString(*username) { logrus.Warn("Username can only contain characters a-z, 0-9, or '_-./='") os.Exit(1) } - pass := getPassword(password, pwdFile, pwdStdin, askPass, os.Stdin) + if len(fmt.Sprintf("@%s:%s", *username, cfg.Global.ServerName)) > 255 { + logrus.Fatalf("Username can not be longer than 255 characters: %s", fmt.Sprintf("@%s:%s", *username, cfg.Global.ServerName)) + } - b := base.NewBaseDendrite(cfg, "create-account") + var pass string + var err error + if !*pwdLess { + pass, err = getPassword(*password, *pwdFile, *pwdStdin, os.Stdin) + if err != nil { + logrus.Fatalln(err) + } + } + + b := base.NewBaseDendrite(cfg, "Monolith") accountDB := b.CreateAccountsDB() accType := api.AccountTypeUser if *isAdmin { accType = api.AccountTypeAdmin } - var err error + + available, err := accountDB.CheckAccountAvailability(context.Background(), *username) + if err != nil { + logrus.Fatalln("Unable check username existence.") + } if *resetPassword { + if available { + logrus.Fatalln("Username could not be found.") + } err = accountDB.SetPassword(context.Background(), *username, pass) if err != nil { logrus.Fatalf("Failed to update password for user %s: %s", *username, err.Error()) @@ -104,6 +125,9 @@ func main() { logrus.Infof("Updated password for user %s and invalidated all logins\n", *username) return } + if !available { + logrus.Fatalln("Username is already in use.") + } _, err = accountDB.CreateAccount(context.Background(), *username, pass, "", accType) if err != nil { @@ -113,53 +137,44 @@ func main() { logrus.Infoln("Created account", *username) } -func getPassword(password, pwdFile *string, pwdStdin, askPass *bool, r io.Reader) string { - // no password option set, use empty password - if password == nil && pwdFile == nil && pwdStdin == nil && askPass == nil { - return "" - } - // password defined as parameter - if password != nil && *password != "" { - return *password - } - +func getPassword(password, pwdFile string, pwdStdin bool, r io.Reader) (string, error) { // read password from file - if pwdFile != nil && *pwdFile != "" { - pw, err := ioutil.ReadFile(*pwdFile) + if pwdFile != "" { + pw, err := ioutil.ReadFile(pwdFile) if err != nil { - logrus.Fatalln("Unable to read password from file:", err) + return "", fmt.Errorf("Unable to read password from file: %v", err) } - return strings.TrimSpace(string(pw)) + return strings.TrimSpace(string(pw)), nil } // read password from stdin - if pwdStdin != nil && *pwdStdin { + if pwdStdin { data, err := ioutil.ReadAll(r) if err != nil { - logrus.Fatalln("Unable to read password from stdin:", err) + return "", fmt.Errorf("Unable to read password from stdin: %v", err) } - return strings.TrimSpace(string(data)) + return strings.TrimSpace(string(data)), nil } - // ask the user to provide the password - if *askPass { + // If no parameter was set, ask the user to provide the password + if password == "" { fmt.Print("Enter Password: ") bytePassword, err := term.ReadPassword(int(os.Stdin.Fd())) if err != nil { - logrus.Fatalln("Unable to read password:", err) + return "", fmt.Errorf("Unable to read password: %v", err) } fmt.Println() fmt.Print("Confirm Password: ") bytePassword2, err := term.ReadPassword(int(os.Stdin.Fd())) if err != nil { - logrus.Fatalln("Unable to read password:", err) + return "", fmt.Errorf("Unable to read password: %v", err) } fmt.Println() if strings.TrimSpace(string(bytePassword)) != strings.TrimSpace(string(bytePassword2)) { - logrus.Fatalln("Entered passwords don't match") + return "", fmt.Errorf("Entered passwords don't match") } - return strings.TrimSpace(string(bytePassword)) + return strings.TrimSpace(string(bytePassword)), nil } - return "" + return password, nil } diff --git a/cmd/create-account/main_test.go b/cmd/create-account/main_test.go index d06eafe46..8e8b946be 100644 --- a/cmd/create-account/main_test.go +++ b/cmd/create-account/main_test.go @@ -8,45 +8,48 @@ import ( func Test_getPassword(t *testing.T) { type args struct { - password *string - pwdFile *string - pwdStdin *bool - askPass *bool + password string + pwdFile string + pwdStdin bool reader io.Reader } pass := "mySecretPass" passwordFile := "testdata/my.pass" - passwordStdin := true reader := &bytes.Buffer{} _, err := reader.WriteString(pass) if err != nil { t.Errorf("unable to write to buffer: %+v", err) } tests := []struct { - name string - args args - want string + name string + args args + want string + wantErr bool }{ - { - name: "no password defined", - args: args{}, - want: "", - }, { name: "password defined", - args: args{password: &pass}, + args: args{ + password: pass, + }, want: pass, }, { name: "pwdFile defined", - args: args{pwdFile: &passwordFile}, + args: args{ + pwdFile: passwordFile, + }, want: pass, }, + { + name: "pwdFile does not exist", + args: args{pwdFile: "iDontExist"}, + wantErr: true, + }, { name: "read pass from stdin defined", args: args{ - pwdStdin: &passwordStdin, + pwdStdin: true, reader: reader, }, want: pass, @@ -54,7 +57,11 @@ func Test_getPassword(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := getPassword(tt.args.password, tt.args.pwdFile, tt.args.pwdStdin, tt.args.askPass, tt.args.reader); got != tt.want { + got, err := getPassword(tt.args.password, tt.args.pwdFile, tt.args.pwdStdin, tt.args.reader) + if !tt.wantErr && err != nil { + t.Errorf("expected no error, but got %v", err) + } + if got != tt.want { t.Errorf("getPassword() = '%v', want '%v'", got, tt.want) } }) From 565b5423ea53bf8b6ab54041ef162c1eba438214 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 25 Mar 2022 13:41:28 +0000 Subject: [PATCH 30/55] One final tweak to the GHA pipeline --- .github/workflows/dendrite.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index e1f71ea2f..735ea9113 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -327,6 +327,7 @@ jobs: name: Integration tests passed needs: [initial-tests-done, upgrade_test, sytest, complement] runs-on: ubuntu-latest + if: ${{ !cancelled() }} # Run this even if prior jobs were skipped steps: - name: Check integration tests passed uses: re-actors/alls-green@release/v1 From 1554d51b375dc35416aa6a8f1b3e436e6a3de6f0 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 25 Mar 2022 14:27:41 +0000 Subject: [PATCH 31/55] Pass DOCKER_TOKEN secret into Docker workflow --- .github/workflows/dendrite.yml | 2 ++ .github/workflows/docker.yml | 3 +++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index 735ea9113..99747f798 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -342,3 +342,5 @@ jobs: if: github.repository == 'matrix-org/dendrite' && github.ref_name == 'main' needs: [integration-tests-done] uses: matrix-org/dendrite/.github/workflows/docker.yml@main + secrets: + DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 129a80487..642587924 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -7,6 +7,9 @@ on: types: [published] workflow_dispatch: # A build was manually requested workflow_call: # Another pipeline called us + secrets: + DOCKER_TOKEN: + required: true env: DOCKER_NAMESPACE: matrixdotorg From 08d995d8094dcc7d4f1de42aadce55e9a441fbcb Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 25 Mar 2022 14:53:06 +0000 Subject: [PATCH 32/55] Version 0.7.0 (#2304) --- CHANGES.md | 38 ++++++++++++++++++++++++++++++++++++++ internal/version.go | 4 ++-- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index eb365baf1..428ecbf2a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,43 @@ # Changelog +## Dendrite 0.7.0 (2022-03-25) + +### Features + +* The roomserver input API will now queue all events into NATS, which provides better crash resilience +* The roomserver input API now configures per-room consumers, which should use less memory +* Canonical aliases can now be added and removed +* MSC2946 Spaces Summary now works correctly, both locally and over federation +* Healthcheck endpoints are now available at: + * `/_dendrite/monitor/up`, which will return 200 when Dendrite is ready to accept requests + * `/_dendrite/monitor/health`, which will return 200 if healthy and 503 if degraded for some reason +* The `X-Matrix` federation authorisation header now includes a `destination` field, as per MSC3383 +* The `/sync` endpoint now uses less memory by only ranging state for rooms that the user has participated in +* The `/messages` endpoint now accepts stream positions in both the `from` and `to` parameters +* Dendrite will now log a warning at startup if the file descriptor limit is set too low +* The federation client will now attempt to use HTTP/2 if available +* The federation client will now attempt to resume TLS sessions if possible, to reduce handshake overheads +* The built-in NATS Server has been updated to version 2.7.4 +* NATS streams that don't match the desired configuration will now be recreated automatically +* When performing a graceful shutdown, Dendrite will now wait for NATS Server to shutdown completely, which should avoid some corruption of data on-disk +* The `create-account` tool has seen a number of improvements, will now ask for passwords automatically + +### Fixes + +* The `/sync` endpoint will no longer lose state events when truncating the timeline for history visibility +* The `/context` endpoint now works correctly with `lazy_load_members` +* The `/directory/list/room/{roomID}` endpoint now correctly reports whether a room is published in the server room directory or not +* Some bugs around appservice username validation have been fixed +* Roomserver output messages are no longer unnecessarily inflated by state events, which should reduce the number of NATS message size errors +* Stream IDs for device list updates are now always 64-bit, which should fix some problems when running Dendrite on a 32-bit system +* Purging room state in the sync API has been fixed after a faulty database query was corrected +* The federation client will now release host records for remote destinations after 5 minutes instead of holding them in memory forever +* Remote media requests will now correctly return an error if the file cannot be found or downloaded +* A panic in the media API that could happen when the remote file doesn't exist has been fixed +* Various bugs around membership state and invites have been fixed +* The memberships table will now be correctly updated when rejecting a federated invite +* The client API and appservice API will now access the user database using the user API rather than accessing the database directly + ## Dendrite 0.6.5 (2022-03-04) ### Features diff --git a/internal/version.go b/internal/version.go index 2e8de4175..cff4d2819 100644 --- a/internal/version.go +++ b/internal/version.go @@ -16,8 +16,8 @@ var build string const ( VersionMajor = 0 - VersionMinor = 6 - VersionPatch = 5 + VersionMinor = 7 + VersionPatch = 0 VersionTag = "" // example: "rc1" ) From ceb3874469eb60bc0ffe816acfddb2b368a48a4f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 28 Mar 2022 10:47:35 +0100 Subject: [PATCH 33/55] Allow stored session parameters to be overwritten in the registration request (#2309) * Allow stored session parameters to be overwritten in the registration request * Remove logging * Close request body * Use `httputil.UnmarshalJSON` as that should enforce UTF-8 correctness * Return `M_NOT_JSON` on read error * Whoops, return the value of `httputil.UnmarshalJSON` * Remove redundant comment --- clientapi/routing/register.go | 51 ++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index af2e99ed0..7d84f2494 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -31,6 +31,7 @@ import ( "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/setup/config" + "github.com/tidwall/gjson" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/tokens" @@ -525,22 +526,37 @@ func Register( userAPI userapi.UserRegisterAPI, cfg *config.ClientAPI, ) util.JSONResponse { + defer req.Body.Close() // nolint: errcheck + reqBody, err := ioutil.ReadAll(req.Body) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.NotJSON("Unable to read request body"), + } + } + var r registerRequest - resErr := httputil.UnmarshalJSONRequest(req, &r) - if resErr != nil { + sessionID := gjson.GetBytes(reqBody, "auth.session").String() + if sessionID == "" { + // Generate a new, random session ID + sessionID = util.RandomString(sessionIDLength) + } else if data, ok := sessions.getParams(sessionID); ok { + // Use the parameters from the session as our defaults. + // Some of these might end up being overwritten if the + // values are specified again in the request body. + r.Username = data.Username + r.Password = data.Password + r.DeviceID = data.DeviceID + r.InitialDisplayName = data.InitialDisplayName + r.InhibitLogin = data.InhibitLogin + } + if resErr := httputil.UnmarshalJSON(reqBody, &r); resErr != nil { return *resErr } if req.URL.Query().Get("kind") == "guest" { return handleGuestRegistration(req, r, cfg, userAPI) } - // Retrieve or generate the sessionID - sessionID := r.Auth.Session - if sessionID == "" { - // Generate a new, random session ID - sessionID = util.RandomString(sessionIDLength) - } - // Don't allow numeric usernames less than MAX_INT64. if _, err := strconv.ParseInt(r.Username, 10, 64); err == nil { return util.JSONResponse{ @@ -568,7 +584,7 @@ func Register( case r.Type == authtypes.LoginTypeApplicationService && accessTokenErr == nil: // Spec-compliant case (the access_token is specified and the login type // is correctly set, so it's an appservice registration) - if resErr = validateApplicationServiceUsername(r.Username); resErr != nil { + if resErr := validateApplicationServiceUsername(r.Username); resErr != nil { return *resErr } case accessTokenErr == nil: @@ -581,11 +597,11 @@ func Register( default: // Spec-compliant case (neither the access_token nor the login type are // specified, so it's a normal user registration) - if resErr = validateUsername(r.Username); resErr != nil { + if resErr := validateUsername(r.Username); resErr != nil { return *resErr } } - if resErr = validatePassword(r.Password); resErr != nil { + if resErr := validatePassword(r.Password); resErr != nil { return *resErr } @@ -835,24 +851,17 @@ func completeRegistration( } }() - if data, ok := sessions.getParams(sessionID); ok { - username = data.Username - password = data.Password - deviceID = data.DeviceID - displayName = data.InitialDisplayName - inhibitLogin = data.InhibitLogin - } if username == "" { return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON("missing username"), + JSON: jsonerror.MissingArgument("Missing username"), } } // Blank passwords are only allowed by registered application services if password == "" && appserviceID == "" { return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON("missing password"), + JSON: jsonerror.MissingArgument("Missing password"), } } var accRes userapi.PerformAccountCreationResponse From 34b9c8c67000bd27db73c987e629c9df0d2c0f28 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 28 Mar 2022 10:55:53 +0100 Subject: [PATCH 34/55] Ensure Dendrite has stopped in Pinecone demo `Stop()` --- build/gobind-pinecone/monolith.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 865457010..ea75d5275 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -401,11 +401,12 @@ func (m *DendriteMonolith) Start() { } func (m *DendriteMonolith) Stop() { + m.processContext.ShutdownDendrite() _ = m.listener.Close() m.PineconeMulticast.Stop() _ = m.PineconeQUIC.Close() - m.processContext.ShutdownDendrite() _ = m.PineconeRouter.Close() + m.processContext.WaitForComponentsToFinish() } const MaxFrameSize = types.MaxFrameSize From 8099bcbc8b19298dc3f392571e53fe86839d6277 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 28 Mar 2022 11:44:25 +0100 Subject: [PATCH 35/55] P2P demo tweaks --- build/gobind-pinecone/monolith.go | 1 + build/gobind-yggdrasil/monolith.go | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index ea75d5275..8ef4bbc71 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -281,6 +281,7 @@ func (m *DendriteMonolith) Start() { cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk)) cfg.Global.PrivateKey = sk cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID) + cfg.Global.JetStream.InMemory = true cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/%s", m.StorageDirectory, prefix)) cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-account.db", m.StorageDirectory, prefix)) cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-mediaapi.db", m.StorageDirectory)) diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go index 3329485aa..a2c9bcff6 100644 --- a/build/gobind-yggdrasil/monolith.go +++ b/build/gobind-yggdrasil/monolith.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/setup" "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" @@ -36,6 +37,7 @@ type DendriteMonolith struct { StorageDirectory string listener net.Listener httpServer *http.Server + processContext *process.ProcessContext } func (m *DendriteMonolith) BaseURL() string { @@ -87,6 +89,7 @@ func (m *DendriteMonolith) Start() { cfg.Global.PrivateKey = ygg.PrivateKey() cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID) cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", m.StorageDirectory)) + cfg.Global.JetStream.InMemory = true cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-account.db", m.StorageDirectory)) cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-mediaapi.db", m.StorageDirectory)) cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-syncapi.db", m.StorageDirectory)) @@ -101,6 +104,7 @@ func (m *DendriteMonolith) Start() { } base := base.NewBaseDendrite(cfg, "Monolith") + m.processContext = base.ProcessContext defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() @@ -197,9 +201,12 @@ func (m *DendriteMonolith) Start() { }() } -func (m *DendriteMonolith) Suspend() { - m.logger.Info("Suspending monolith") +func (m *DendriteMonolith) Stop() { if err := m.httpServer.Close(); err != nil { m.logger.Warn("Error stopping HTTP server:", err) } + if m.processContext != nil { + m.processContext.ShutdownDendrite() + m.processContext.WaitForComponentsToFinish() + } } From 0692be44d91a42945dde0eab3e2f7481cc2e0896 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 28 Mar 2022 13:31:17 +0100 Subject: [PATCH 36/55] Fix account availability on register --- userapi/internal/api.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/userapi/internal/api.go b/userapi/internal/api.go index afe57da25..206c6f7de 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -776,12 +776,8 @@ func (a *UserInternalAPI) QueryNumericLocalpart(ctx context.Context, res *api.Qu } func (a *UserInternalAPI) QueryAccountAvailability(ctx context.Context, req *api.QueryAccountAvailabilityRequest, res *api.QueryAccountAvailabilityResponse) error { - _, err := a.DB.CheckAccountAvailability(ctx, req.Localpart) - if err == sql.ErrNoRows { - res.Available = true - return nil - } - res.Available = false + var err error + res.Available, err = a.DB.CheckAccountAvailability(ctx, req.Localpart) return err } From 7972915806348847ecd9a9b8a1b1ff0609cb883c Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 28 Mar 2022 16:25:26 +0100 Subject: [PATCH 37/55] User directory for nearby Pinecone peers (P2P demo) (#2311) * User directory for nearby Pinecone peers * Fix mux routing * Use config to determine which server notices user to exclude --- build/gobind-pinecone/monolith.go | 21 ++- clientapi/auth/authtypes/profile.go | 1 + clientapi/clientapi.go | 3 +- clientapi/routing/routing.go | 2 + clientapi/routing/userdirectory.go | 14 +- cmd/dendrite-demo-pinecone/main.go | 21 ++- cmd/dendrite-demo-pinecone/users/users.go | 145 ++++++++++++++++++ .../personalities/clientapi.go | 2 +- setup/base/base.go | 1 + setup/monolith.go | 9 +- userapi/api/api.go | 4 + userapi/storage/postgres/profile_table.go | 11 +- userapi/storage/postgres/storage.go | 4 +- userapi/storage/sqlite3/profile_table.go | 10 +- userapi/storage/sqlite3/storage.go | 4 +- userapi/storage/storage.go | 6 +- userapi/storage/storage_wasm.go | 3 +- userapi/userapi_test.go | 2 +- 18 files changed, 226 insertions(+), 37 deletions(-) create mode 100644 cmd/dendrite-demo-pinecone/users/users.go diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 8ef4bbc71..346c5d1ec 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/conn" "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms" + "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/users" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" @@ -326,6 +327,9 @@ func (m *DendriteMonolith) Start() { // This is different to rsAPI which can be the http client which doesn't need this dependency rsAPI.SetFederationAPI(fsAPI, keyRing) + userProvider := users.NewPineconeUserProvider(m.PineconeRouter, m.PineconeQUIC, m.userAPI, federation) + roomProvider := rooms.NewPineconeRoomProvider(m.PineconeRouter, m.PineconeQUIC, fsAPI, federation) + monolith := setup.Monolith{ Config: base.Cfg, AccountDB: accountDB, @@ -333,13 +337,14 @@ func (m *DendriteMonolith) Start() { FedClient: federation, KeyRing: keyRing, - AppserviceAPI: asAPI, - EDUInternalAPI: eduInputAPI, - FederationAPI: fsAPI, - RoomserverAPI: rsAPI, - UserAPI: m.userAPI, - KeyAPI: keyAPI, - ExtPublicRoomsProvider: rooms.NewPineconeRoomProvider(m.PineconeRouter, m.PineconeQUIC, fsAPI, federation), + AppserviceAPI: asAPI, + EDUInternalAPI: eduInputAPI, + FederationAPI: fsAPI, + RoomserverAPI: rsAPI, + UserAPI: m.userAPI, + KeyAPI: keyAPI, + ExtPublicRoomsProvider: roomProvider, + ExtUserDirectoryProvider: userProvider, } monolith.AddAllPublicRoutes( base.ProcessContext, @@ -357,10 +362,12 @@ func (m *DendriteMonolith) Start() { httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux) pMux := mux.NewRouter().SkipClean(true).UseEncodedPath() + pMux.PathPrefix(users.PublicURL).HandlerFunc(userProvider.FederatedUserProfiles) pMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux) pMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux) pHTTP := m.PineconeQUIC.HTTP() + pHTTP.Mux().Handle(users.PublicURL, pMux) pHTTP.Mux().Handle(httputil.PublicFederationPathPrefix, pMux) pHTTP.Mux().Handle(httputil.PublicMediaPathPrefix, pMux) diff --git a/clientapi/auth/authtypes/profile.go b/clientapi/auth/authtypes/profile.go index 902850bc0..29468c168 100644 --- a/clientapi/auth/authtypes/profile.go +++ b/clientapi/auth/authtypes/profile.go @@ -17,6 +17,7 @@ package authtypes // Profile represents the profile for a Matrix account. type Profile struct { Localpart string `json:"local_part"` + ServerName string `json:"server_name,omitempty"` // NOTSPEC: only set by Pinecone user provider DisplayName string `json:"display_name"` AvatarURL string `json:"avatar_url"` } diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 31a53a706..d0ef368d1 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -45,6 +45,7 @@ func AddPublicRoutes( transactionsCache *transactions.Cache, fsAPI federationAPI.FederationInternalAPI, userAPI userapi.UserInternalAPI, + userDirectoryProvider userapi.UserDirectoryProvider, keyAPI keyserverAPI.KeyInternalAPI, extRoomsProvider api.ExtraPublicRoomsProvider, mscCfg *config.MSCs, @@ -58,7 +59,7 @@ func AddPublicRoutes( routing.Setup( router, synapseAdminRouter, cfg, eduInputAPI, rsAPI, asAPI, - userAPI, federation, + userAPI, userDirectoryProvider, federation, syncProducer, transactionsCache, fsAPI, keyAPI, extRoomsProvider, mscCfg, ) diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index e173831ab..218087bb3 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -51,6 +51,7 @@ func Setup( rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, userAPI userapi.UserInternalAPI, + userDirectoryProvider userapi.UserDirectoryProvider, federation *gomatrixserverlib.FederationClient, syncProducer *producers.SyncAPIProducer, transactionsCache *transactions.Cache, @@ -904,6 +905,7 @@ func Setup( device, userAPI, rsAPI, + userDirectoryProvider, cfg.Matrix.ServerName, postContent.SearchString, postContent.Limit, diff --git a/clientapi/routing/userdirectory.go b/clientapi/routing/userdirectory.go index 2659bc9cc..ab73cf430 100644 --- a/clientapi/routing/userdirectory.go +++ b/clientapi/routing/userdirectory.go @@ -16,6 +16,7 @@ package routing import ( "context" + "database/sql" "fmt" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" @@ -35,6 +36,7 @@ func SearchUserDirectory( device *userapi.Device, userAPI userapi.UserInternalAPI, rsAPI api.RoomserverInternalAPI, + provider userapi.UserDirectoryProvider, serverName gomatrixserverlib.ServerName, searchString string, limit int, @@ -50,13 +52,12 @@ func SearchUserDirectory( } // First start searching local users. - userReq := &userapi.QuerySearchProfilesRequest{ SearchString: searchString, Limit: limit, } userRes := &userapi.QuerySearchProfilesResponse{} - if err := userAPI.QuerySearchProfiles(ctx, userReq, userRes); err != nil { + if err := provider.QuerySearchProfiles(ctx, userReq, userRes); err != nil { errRes := util.ErrorResponse(fmt.Errorf("userAPI.QuerySearchProfiles: %w", err)) return &errRes } @@ -67,7 +68,12 @@ func SearchUserDirectory( break } - userID := fmt.Sprintf("@%s:%s", user.Localpart, serverName) + var userID string + if user.ServerName != "" { + userID = fmt.Sprintf("@%s:%s", user.Localpart, user.ServerName) + } else { + userID = fmt.Sprintf("@%s:%s", user.Localpart, serverName) + } if _, ok := results[userID]; !ok { results[userID] = authtypes.FullyQualifiedProfile{ UserID: userID, @@ -87,7 +93,7 @@ func SearchUserDirectory( Limit: limit - len(results), } stateRes := &api.QueryKnownUsersResponse{} - if err := rsAPI.QueryKnownUsers(ctx, stateReq, stateRes); err != nil { + if err := rsAPI.QueryKnownUsers(ctx, stateReq, stateRes); err != nil && err != sql.ErrNoRows { errRes := util.ErrorResponse(fmt.Errorf("rsAPI.QueryKnownUsers: %w", err)) return &errRes } diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index 45f186985..87054dc82 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -35,6 +35,7 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/conn" "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/embed" "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms" + "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/users" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" @@ -198,6 +199,9 @@ func main() { rsComponent.SetFederationAPI(fsAPI, keyRing) + userProvider := users.NewPineconeUserProvider(pRouter, pQUIC, userAPI, federation) + roomProvider := rooms.NewPineconeRoomProvider(pRouter, pQUIC, fsAPI, federation) + monolith := setup.Monolith{ Config: base.Cfg, AccountDB: accountDB, @@ -205,13 +209,14 @@ func main() { FedClient: federation, KeyRing: keyRing, - AppserviceAPI: asAPI, - EDUInternalAPI: eduInputAPI, - FederationAPI: fsAPI, - RoomserverAPI: rsAPI, - UserAPI: userAPI, - KeyAPI: keyAPI, - ExtPublicRoomsProvider: rooms.NewPineconeRoomProvider(pRouter, pQUIC, fsAPI, federation), + AppserviceAPI: asAPI, + EDUInternalAPI: eduInputAPI, + FederationAPI: fsAPI, + RoomserverAPI: rsAPI, + UserAPI: userAPI, + KeyAPI: keyAPI, + ExtPublicRoomsProvider: roomProvider, + ExtUserDirectoryProvider: userProvider, } monolith.AddAllPublicRoutes( base.ProcessContext, @@ -250,10 +255,12 @@ func main() { embed.Embed(httpRouter, *instancePort, "Pinecone Demo") pMux := mux.NewRouter().SkipClean(true).UseEncodedPath() + pMux.PathPrefix(users.PublicURL).HandlerFunc(userProvider.FederatedUserProfiles) pMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux) pMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux) pHTTP := pQUIC.HTTP() + pHTTP.Mux().Handle(users.PublicURL, pMux) pHTTP.Mux().Handle(httputil.PublicFederationPathPrefix, pMux) pHTTP.Mux().Handle(httputil.PublicMediaPathPrefix, pMux) diff --git a/cmd/dendrite-demo-pinecone/users/users.go b/cmd/dendrite-demo-pinecone/users/users.go new file mode 100644 index 000000000..ffbd27ee9 --- /dev/null +++ b/cmd/dendrite-demo-pinecone/users/users.go @@ -0,0 +1,145 @@ +package users + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "sync" + "time" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + clienthttputil "github.com/matrix-org/dendrite/clientapi/httputil" + userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + + pineconeRouter "github.com/matrix-org/pinecone/router" + pineconeSessions "github.com/matrix-org/pinecone/sessions" +) + +type PineconeUserProvider struct { + r *pineconeRouter.Router + s *pineconeSessions.Sessions + userAPI userapi.UserProfileAPI + fedClient *gomatrixserverlib.FederationClient +} + +const PublicURL = "/_matrix/p2p/profiles" + +func NewPineconeUserProvider( + r *pineconeRouter.Router, + s *pineconeSessions.Sessions, + userAPI userapi.UserProfileAPI, + fedClient *gomatrixserverlib.FederationClient, +) *PineconeUserProvider { + p := &PineconeUserProvider{ + r: r, + s: s, + userAPI: userAPI, + fedClient: fedClient, + } + return p +} + +func (p *PineconeUserProvider) FederatedUserProfiles(w http.ResponseWriter, r *http.Request) { + req := &userapi.QuerySearchProfilesRequest{Limit: 25} + res := &userapi.QuerySearchProfilesResponse{} + if err := clienthttputil.UnmarshalJSONRequest(r, &req); err != nil { + w.WriteHeader(400) + return + } + if err := p.userAPI.QuerySearchProfiles(r.Context(), req, res); err != nil { + w.WriteHeader(400) + return + } + j, err := json.Marshal(res) + if err != nil { + w.WriteHeader(400) + return + } + w.WriteHeader(200) + _, _ = w.Write(j) +} + +func (p *PineconeUserProvider) QuerySearchProfiles(ctx context.Context, req *userapi.QuerySearchProfilesRequest, res *userapi.QuerySearchProfilesResponse) error { + list := map[string]struct{}{} + for _, k := range p.r.Peers() { + list[k.PublicKey] = struct{}{} + } + res.Profiles = bulkFetchUserDirectoriesFromServers(context.Background(), req, p.fedClient, list) + return nil +} + +// bulkFetchUserDirectoriesFromServers fetches users from the list of homeservers. +// Returns a list of user profiles. +func bulkFetchUserDirectoriesFromServers( + ctx context.Context, req *userapi.QuerySearchProfilesRequest, + fedClient *gomatrixserverlib.FederationClient, + homeservers map[string]struct{}, +) (profiles []authtypes.Profile) { + jsonBody, err := json.Marshal(req) + if err != nil { + return nil + } + + limit := 200 + // follow pipeline semantics, see https://blog.golang.org/pipelines for more info. + // goroutines send rooms to this channel + profileCh := make(chan authtypes.Profile, int(limit)) + // signalling channel to tell goroutines to stop sending rooms and quit + done := make(chan bool) + // signalling to say when we can close the room channel + var wg sync.WaitGroup + wg.Add(len(homeservers)) + // concurrently query for public rooms + reqctx, reqcancel := context.WithTimeout(ctx, time.Second*5) + for hs := range homeservers { + go func(homeserverDomain string) { + defer wg.Done() + util.GetLogger(reqctx).WithField("hs", homeserverDomain).Info("Querying HS for users") + + jsonBodyReader := bytes.NewBuffer(jsonBody) + httpReq, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("matrix://%s%s", homeserverDomain, PublicURL), jsonBodyReader) + if err != nil { + util.GetLogger(reqctx).WithError(err).WithField("hs", homeserverDomain).Warn( + "bulkFetchUserDirectoriesFromServers: failed to create request", + ) + } + res := &userapi.QuerySearchProfilesResponse{} + if err = fedClient.DoRequestAndParseResponse(reqctx, httpReq, res); err != nil { + util.GetLogger(reqctx).WithError(err).WithField("hs", homeserverDomain).Warn( + "bulkFetchUserDirectoriesFromServers: failed to query hs", + ) + return + } + for _, profile := range res.Profiles { + profile.ServerName = homeserverDomain + // atomically send a room or stop + select { + case profileCh <- profile: + case <-done: + case <-reqctx.Done(): + util.GetLogger(reqctx).WithError(err).WithField("hs", homeserverDomain).Info("Interrupted whilst sending profiles") + return + } + } + }(hs) + } + + select { + case <-time.After(5 * time.Second): + default: + wg.Wait() + } + reqcancel() + close(done) + close(profileCh) + + for profile := range profileCh { + profiles = append(profiles, profile) + } + + return profiles +} diff --git a/cmd/dendrite-polylith-multi/personalities/clientapi.go b/cmd/dendrite-polylith-multi/personalities/clientapi.go index a2036de35..978d8b0a4 100644 --- a/cmd/dendrite-polylith-multi/personalities/clientapi.go +++ b/cmd/dendrite-polylith-multi/personalities/clientapi.go @@ -33,7 +33,7 @@ func ClientAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { clientapi.AddPublicRoutes( base.ProcessContext, base.PublicClientAPIMux, base.SynapseAdminMux, &base.Cfg.ClientAPI, - federation, rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, userAPI, + federation, rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, userAPI, userAPI, keyAPI, nil, &cfg.MSCs, ) diff --git a/setup/base/base.go b/setup/base/base.go index c3ed05416..6135e080e 100644 --- a/setup/base/base.go +++ b/setup/base/base.go @@ -289,6 +289,7 @@ func (b *BaseDendrite) CreateAccountsDB() userdb.Database { b.Cfg.UserAPI.BCryptCost, b.Cfg.UserAPI.OpenIDTokenLifetimeMS, userapi.DefaultLoginTokenLifetime, + b.Cfg.Global.ServerNotices.LocalPart, ) if err != nil { logrus.WithError(err).Panicf("failed to connect to accounts db") diff --git a/setup/monolith.go b/setup/monolith.go index 8e6240d37..cf6872f9c 100644 --- a/setup/monolith.go +++ b/setup/monolith.go @@ -51,16 +51,21 @@ type Monolith struct { KeyAPI keyAPI.KeyInternalAPI // Optional - ExtPublicRoomsProvider api.ExtraPublicRoomsProvider + ExtPublicRoomsProvider api.ExtraPublicRoomsProvider + ExtUserDirectoryProvider userapi.UserDirectoryProvider } // AddAllPublicRoutes attaches all public paths to the given router func (m *Monolith) AddAllPublicRoutes(process *process.ProcessContext, csMux, ssMux, keyMux, wkMux, mediaMux, synapseMux *mux.Router) { + userDirectoryProvider := m.ExtUserDirectoryProvider + if userDirectoryProvider == nil { + userDirectoryProvider = m.UserAPI + } clientapi.AddPublicRoutes( process, csMux, synapseMux, &m.Config.ClientAPI, m.FedClient, m.RoomserverAPI, m.EDUInternalAPI, m.AppserviceAPI, transactions.New(), - m.FederationAPI, m.UserAPI, m.KeyAPI, + m.FederationAPI, m.UserAPI, userDirectoryProvider, m.KeyAPI, m.ExtPublicRoomsProvider, &m.Config.MSCs, ) federationapi.AddPublicRoutes( diff --git a/userapi/api/api.go b/userapi/api/api.go index 513a060c4..a9544f00d 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -54,6 +54,10 @@ type UserInternalAPI interface { QueryNotifications(ctx context.Context, req *QueryNotificationsRequest, res *QueryNotificationsResponse) error } +type UserDirectoryProvider interface { + QuerySearchProfiles(ctx context.Context, req *QuerySearchProfilesRequest, res *QuerySearchProfilesResponse) error +} + // UserProfileAPI provides functions for getting user profiles type UserProfileAPI interface { QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error diff --git a/userapi/storage/postgres/profile_table.go b/userapi/storage/postgres/profile_table.go index 32a4b5506..6d336eb8e 100644 --- a/userapi/storage/postgres/profile_table.go +++ b/userapi/storage/postgres/profile_table.go @@ -53,6 +53,7 @@ const selectProfilesBySearchSQL = "" + "SELECT localpart, display_name, avatar_url FROM account_profiles WHERE localpart LIKE $1 OR display_name LIKE $1 LIMIT $2" type profilesStatements struct { + serverNoticesLocalpart string insertProfileStmt *sql.Stmt selectProfileByLocalpartStmt *sql.Stmt setAvatarURLStmt *sql.Stmt @@ -60,8 +61,10 @@ type profilesStatements struct { selectProfilesBySearchStmt *sql.Stmt } -func NewPostgresProfilesTable(db *sql.DB) (tables.ProfileTable, error) { - s := &profilesStatements{} +func NewPostgresProfilesTable(db *sql.DB, serverNoticesLocalpart string) (tables.ProfileTable, error) { + s := &profilesStatements{ + serverNoticesLocalpart: serverNoticesLocalpart, + } _, err := db.Exec(profilesSchema) if err != nil { return nil, err @@ -126,7 +129,9 @@ func (s *profilesStatements) SelectProfilesBySearch( if err := rows.Scan(&profile.Localpart, &profile.DisplayName, &profile.AvatarURL); err != nil { return nil, err } - profiles = append(profiles, profile) + if profile.Localpart != s.serverNoticesLocalpart { + profiles = append(profiles, profile) + } } return profiles, nil } diff --git a/userapi/storage/postgres/storage.go b/userapi/storage/postgres/storage.go index c74a999f4..b2a517605 100644 --- a/userapi/storage/postgres/storage.go +++ b/userapi/storage/postgres/storage.go @@ -30,7 +30,7 @@ import ( ) // NewDatabase creates a new accounts and profiles database -func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration) (*shared.Database, error) { +func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (*shared.Database, error) { db, err := sqlutil.Open(dbProperties) if err != nil { return nil, err @@ -77,7 +77,7 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver if err != nil { return nil, fmt.Errorf("NewPostgresOpenIDTable: %w", err) } - profilesTable, err := NewPostgresProfilesTable(db) + profilesTable, err := NewPostgresProfilesTable(db, serverNoticesLocalpart) if err != nil { return nil, fmt.Errorf("NewPostgresProfilesTable: %w", err) } diff --git a/userapi/storage/sqlite3/profile_table.go b/userapi/storage/sqlite3/profile_table.go index d85b19c7b..3050ff4b5 100644 --- a/userapi/storage/sqlite3/profile_table.go +++ b/userapi/storage/sqlite3/profile_table.go @@ -54,6 +54,7 @@ const selectProfilesBySearchSQL = "" + type profilesStatements struct { db *sql.DB + serverNoticesLocalpart string insertProfileStmt *sql.Stmt selectProfileByLocalpartStmt *sql.Stmt setAvatarURLStmt *sql.Stmt @@ -61,9 +62,10 @@ type profilesStatements struct { selectProfilesBySearchStmt *sql.Stmt } -func NewSQLiteProfilesTable(db *sql.DB) (tables.ProfileTable, error) { +func NewSQLiteProfilesTable(db *sql.DB, serverNoticesLocalpart string) (tables.ProfileTable, error) { s := &profilesStatements{ - db: db, + db: db, + serverNoticesLocalpart: serverNoticesLocalpart, } _, err := db.Exec(profilesSchema) if err != nil { @@ -131,7 +133,9 @@ func (s *profilesStatements) SelectProfilesBySearch( if err := rows.Scan(&profile.Localpart, &profile.DisplayName, &profile.AvatarURL); err != nil { return nil, err } - profiles = append(profiles, profile) + if profile.Localpart != s.serverNoticesLocalpart { + profiles = append(profiles, profile) + } } return profiles, nil } diff --git a/userapi/storage/sqlite3/storage.go b/userapi/storage/sqlite3/storage.go index b5bb96c42..03c013f00 100644 --- a/userapi/storage/sqlite3/storage.go +++ b/userapi/storage/sqlite3/storage.go @@ -31,7 +31,7 @@ import ( ) // NewDatabase creates a new accounts and profiles database -func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration) (*shared.Database, error) { +func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (*shared.Database, error) { db, err := sqlutil.Open(dbProperties) if err != nil { return nil, err @@ -78,7 +78,7 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver if err != nil { return nil, fmt.Errorf("NewSQLiteOpenIDTable: %w", err) } - profilesTable, err := NewSQLiteProfilesTable(db) + profilesTable, err := NewSQLiteProfilesTable(db, serverNoticesLocalpart) if err != nil { return nil, fmt.Errorf("NewSQLiteProfilesTable: %w", err) } diff --git a/userapi/storage/storage.go b/userapi/storage/storage.go index 4711439af..f372fe7dc 100644 --- a/userapi/storage/storage.go +++ b/userapi/storage/storage.go @@ -30,12 +30,12 @@ import ( // NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) // and sets postgres connection parameters -func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration) (Database, error) { +func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (Database, error) { switch { case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.NewDatabase(dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime) + return sqlite3.NewDatabase(dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart) case dbProperties.ConnectionString.IsPostgres(): - return postgres.NewDatabase(dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime) + return postgres.NewDatabase(dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart) default: return nil, fmt.Errorf("unexpected database type") } diff --git a/userapi/storage/storage_wasm.go b/userapi/storage/storage_wasm.go index 701dcd833..779f77568 100644 --- a/userapi/storage/storage_wasm.go +++ b/userapi/storage/storage_wasm.go @@ -29,10 +29,11 @@ func NewDatabase( bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, + serverNoticesLocalpart string, ) (Database, error) { switch { case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.NewDatabase(dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime) + return sqlite3.NewDatabase(dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart) case dbProperties.ConnectionString.IsPostgres(): return nil, fmt.Errorf("can't use Postgres implementation") default: diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go index 25319c4bf..8c3608bd8 100644 --- a/userapi/userapi_test.go +++ b/userapi/userapi_test.go @@ -52,7 +52,7 @@ func MustMakeInternalAPI(t *testing.T, opts apiTestOpts) (api.UserInternalAPI, s MaxOpenConnections: 1, MaxIdleConnections: 1, } - accountDB, err := storage.NewDatabase(dbopts, serverName, bcrypt.MinCost, config.DefaultOpenIDTokenLifetimeMS, opts.loginTokenLifetime) + accountDB, err := storage.NewDatabase(dbopts, serverName, bcrypt.MinCost, config.DefaultOpenIDTokenLifetimeMS, opts.loginTokenLifetime, "") if err != nil { t.Fatalf("failed to create account DB: %s", err) } From 49dc49b232432d52c082646cc6f778593f4cb8b4 Mon Sep 17 00:00:00 2001 From: S7evinK <2353100+S7evinK@users.noreply.github.com> Date: Tue, 29 Mar 2022 14:14:35 +0200 Subject: [PATCH 38/55] Remove eduserver (#2306) * Move receipt sending to own JetStream producer * Move SendToDevice to producer * Remove most parts of the EDU server * Fix SendToDevice & copyrights * Move structs, cleanup EDU Server traces * Use HeadersOnly subscription * Missing file * Fix linter issues * Move consumers to own files * Rename durable consumer; Consumer cleanup * Docs/config cleanup --- build/docker/config/dendrite.yaml | 6 - build/docker/docker-compose.polylith.yml | 12 - build/gobind-pinecone/monolith.go | 7 - build/gobind-yggdrasil/monolith.go | 17 +- clientapi/clientapi.go | 13 +- clientapi/producers/syncapi.go | 124 ++++++++- clientapi/routing/account_data.go | 5 +- clientapi/routing/receipt.go | 11 +- clientapi/routing/routing.go | 12 +- clientapi/routing/sendtodevice.go | 10 +- clientapi/routing/sendtyping.go | 11 +- cmd/dendrite-demo-libp2p/main.go | 7 - cmd/dendrite-demo-pinecone/main.go | 7 - cmd/dendrite-demo-yggdrasil/main.go | 17 +- cmd/dendrite-monolith-server/main.go | 21 +- cmd/dendrite-polylith-multi/main.go | 1 - .../personalities/clientapi.go | 3 +- .../personalities/eduserver.go | 33 --- .../personalities/federationapi.go | 4 +- cmd/dendritejs-pinecone/main.go | 14 +- cmd/dendritejs/main.go | 4 - dendrite-config.yaml | 6 - docs/INSTALL.md | 8 - eduserver/api/input.go | 103 ------- eduserver/api/output.go | 57 ---- eduserver/api/types.go | 59 ---- eduserver/api/wrapper.go | 88 ------ eduserver/eduserver.go | 56 ---- eduserver/input/input.go | 198 -------------- eduserver/inthttp/client.go | 70 ----- eduserver/inthttp/server.go | 54 ---- federationapi/consumers/eduserver.go | 257 ------------------ federationapi/consumers/keychange.go | 4 +- federationapi/consumers/receipts.go | 141 ++++++++++ federationapi/consumers/sendtodevice.go | 125 +++++++++ federationapi/consumers/typing.go | 119 ++++++++ federationapi/federationapi.go | 41 ++- federationapi/federationapi_test.go | 2 +- federationapi/producers/syncapi.go | 144 ++++++++++ federationapi/routing/routing.go | 6 +- federationapi/routing/send.go | 39 +-- federationapi/routing/send_test.go | 40 --- federationapi/types/types.go | 15 + .../caching/cache_typing.go | 6 +- .../caching/cache_typing_test.go | 6 +- internal/test/config.go | 2 - keyserver/api/api.go | 18 +- keyserver/internal/cross_signing.go | 5 +- keyserver/producers/keychange.go | 5 +- setup/base/base.go | 11 - setup/config/config.go | 15 +- setup/config/config_eduserver.go | 17 -- setup/config/config_test.go | 4 - setup/jetstream/nats.go | 21 ++ setup/jetstream/streams.go | 5 +- setup/monolith.go | 18 +- syncapi/consumers/clientapi.go | 2 +- .../{eduserver_receipts.go => receipts.go} | 24 +- ...server_sendtodevice.go => sendtodevice.go} | 26 +- .../{eduserver_typing.go => typing.go} | 51 ++-- syncapi/storage/interface.go | 5 +- syncapi/storage/postgres/receipt_table.go | 7 +- syncapi/storage/shared/syncserver.go | 5 +- syncapi/storage/sqlite3/receipt_table.go | 7 +- syncapi/storage/tables/interface.go | 3 +- syncapi/streams/stream_receipt.go | 19 +- syncapi/streams/stream_typing.go | 4 +- syncapi/streams/streams.go | 4 +- syncapi/syncapi.go | 6 +- syncapi/types/types.go | 18 ++ 70 files changed, 931 insertions(+), 1354 deletions(-) delete mode 100644 cmd/dendrite-polylith-multi/personalities/eduserver.go delete mode 100644 eduserver/api/input.go delete mode 100644 eduserver/api/output.go delete mode 100644 eduserver/api/types.go delete mode 100644 eduserver/api/wrapper.go delete mode 100644 eduserver/eduserver.go delete mode 100644 eduserver/input/input.go delete mode 100644 eduserver/inthttp/client.go delete mode 100644 eduserver/inthttp/server.go delete mode 100644 federationapi/consumers/eduserver.go create mode 100644 federationapi/consumers/receipts.go create mode 100644 federationapi/consumers/sendtodevice.go create mode 100644 federationapi/consumers/typing.go create mode 100644 federationapi/producers/syncapi.go rename eduserver/cache/cache.go => internal/caching/cache_typing.go (97%) rename eduserver/cache/cache_test.go => internal/caching/cache_typing_test.go (97%) delete mode 100644 setup/config/config_eduserver.go rename syncapi/consumers/{eduserver_receipts.go => receipts.go} (87%) rename syncapi/consumers/{eduserver_sendtodevice.go => sendtodevice.go} (87%) rename syncapi/consumers/{eduserver_typing.go => typing.go} (67%) diff --git a/build/docker/config/dendrite.yaml b/build/docker/config/dendrite.yaml index c01ab62e7..f3d373035 100644 --- a/build/docker/config/dendrite.yaml +++ b/build/docker/config/dendrite.yaml @@ -160,12 +160,6 @@ client_api: threshold: 5 cooloff_ms: 500 -# Configuration for the EDU server. -edu_server: - internal_api: - listen: http://0.0.0.0:7778 - connect: http://edu_server:7778 - # Configuration for the Federation API. federation_api: internal_api: diff --git a/build/docker/docker-compose.polylith.yml b/build/docker/docker-compose.polylith.yml index 207d0451a..de0ab0aa2 100644 --- a/build/docker/docker-compose.polylith.yml +++ b/build/docker/docker-compose.polylith.yml @@ -84,18 +84,6 @@ services: - internal restart: unless-stopped - edu_server: - hostname: edu_server - image: matrixdotorg/dendrite-polylith:latest - command: eduserver - volumes: - - ./config:/etc/dendrite - depends_on: - - jetstream - networks: - - internal - restart: unless-stopped - federation_api: hostname: federation_api image: matrixdotorg/dendrite-polylith:latest diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 346c5d1ec..efc21f596 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -24,8 +24,6 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms" "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/users" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" - "github.com/matrix-org/dendrite/eduserver" - "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/httputil" @@ -317,10 +315,6 @@ func (m *DendriteMonolith) Start() { m.userAPI = userapi.NewInternalAPI(base, accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, base.PushGatewayHTTPClient()) keyAPI.SetUserAPI(m.userAPI) - eduInputAPI := eduserver.NewInternalAPI( - base, cache.New(), m.userAPI, - ) - asAPI := appservice.NewInternalAPI(base, m.userAPI, rsAPI) // The underlying roomserver implementation needs to be able to call the fedsender. @@ -338,7 +332,6 @@ func (m *DendriteMonolith) Start() { KeyRing: keyRing, AppserviceAPI: asAPI, - EDUInternalAPI: eduInputAPI, FederationAPI: fsAPI, RoomserverAPI: rsAPI, UserAPI: m.userAPI, diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go index a2c9bcff6..87dcad2e8 100644 --- a/build/gobind-yggdrasil/monolith.go +++ b/build/gobind-yggdrasil/monolith.go @@ -13,8 +13,6 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggrooms" - "github.com/matrix-org/dendrite/eduserver" - "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/httputil" @@ -123,10 +121,6 @@ func (m *DendriteMonolith) Start() { userAPI := userapi.NewInternalAPI(base, accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, base.PushGatewayHTTPClient()) keyAPI.SetUserAPI(userAPI) - eduInputAPI := eduserver.NewInternalAPI( - base, cache.New(), userAPI, - ) - asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) @@ -141,12 +135,11 @@ func (m *DendriteMonolith) Start() { FedClient: federation, KeyRing: keyRing, - AppserviceAPI: asAPI, - EDUInternalAPI: eduInputAPI, - FederationAPI: fsAPI, - RoomserverAPI: rsAPI, - UserAPI: userAPI, - KeyAPI: keyAPI, + AppserviceAPI: asAPI, + FederationAPI: fsAPI, + RoomserverAPI: rsAPI, + UserAPI: userAPI, + KeyAPI: keyAPI, ExtPublicRoomsProvider: yggrooms.NewYggdrasilRoomProvider( ygg, fsAPI, federation, ), diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index d0ef368d1..d4b417a31 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -20,7 +20,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/api" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/clientapi/routing" - eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/transactions" keyserverAPI "github.com/matrix-org/dendrite/keyserver/api" @@ -40,7 +39,6 @@ func AddPublicRoutes( cfg *config.ClientAPI, federation *gomatrixserverlib.FederationClient, rsAPI roomserverAPI.RoomserverInternalAPI, - eduInputAPI eduServerAPI.EDUServerInputAPI, asAPI appserviceAPI.AppServiceQueryAPI, transactionsCache *transactions.Cache, fsAPI federationAPI.FederationInternalAPI, @@ -53,12 +51,17 @@ func AddPublicRoutes( js, _ := jetstream.Prepare(process, &cfg.Matrix.JetStream) syncProducer := &producers.SyncAPIProducer{ - JetStream: js, - Topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputClientData), + JetStream: js, + TopicClientData: cfg.Matrix.JetStream.Prefixed(jetstream.OutputClientData), + TopicReceiptEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent), + TopicSendToDeviceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), + TopicTypingEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent), + UserAPI: userAPI, + ServerName: cfg.Matrix.ServerName, } routing.Setup( - router, synapseAdminRouter, cfg, eduInputAPI, rsAPI, asAPI, + router, synapseAdminRouter, cfg, rsAPI, asAPI, userAPI, userDirectoryProvider, federation, syncProducer, transactionsCache, fsAPI, keyAPI, extRoomsProvider, mscCfg, diff --git a/clientapi/producers/syncapi.go b/clientapi/producers/syncapi.go index 9ab90391d..2dee04e3a 100644 --- a/clientapi/producers/syncapi.go +++ b/clientapi/producers/syncapi.go @@ -15,24 +15,34 @@ package producers import ( + "context" "encoding/json" + "strconv" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/syncapi/types" + userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" "github.com/nats-io/nats.go" log "github.com/sirupsen/logrus" ) // SyncAPIProducer produces events for the sync API server to consume type SyncAPIProducer struct { - Topic string - JetStream nats.JetStreamContext + TopicClientData string + TopicReceiptEvent string + TopicSendToDeviceEvent string + TopicTypingEvent string + JetStream nats.JetStreamContext + ServerName gomatrixserverlib.ServerName + UserAPI userapi.UserInternalAPI } // SendData sends account data to the sync API server func (p *SyncAPIProducer) SendData(userID string, roomID string, dataType string, readMarker *eventutil.ReadMarkerJSON) error { m := &nats.Msg{ - Subject: p.Topic, + Subject: p.TopicClientData, Header: nats.Header{}, } m.Header.Set(jetstream.UserID, userID) @@ -52,8 +62,114 @@ func (p *SyncAPIProducer) SendData(userID string, roomID string, dataType string "user_id": userID, "room_id": roomID, "data_type": dataType, - }).Tracef("Producing to topic '%s'", p.Topic) + }).Tracef("Producing to topic '%s'", p.TopicClientData) _, err = p.JetStream.PublishMsg(m) return err } + +func (p *SyncAPIProducer) SendReceipt( + ctx context.Context, + userID, roomID, eventID, receiptType string, timestamp gomatrixserverlib.Timestamp, +) error { + m := &nats.Msg{ + Subject: p.TopicReceiptEvent, + Header: nats.Header{}, + } + m.Header.Set(jetstream.UserID, userID) + m.Header.Set(jetstream.RoomID, roomID) + m.Header.Set(jetstream.EventID, eventID) + m.Header.Set("type", receiptType) + m.Header.Set("timestamp", strconv.Itoa(int(timestamp))) + + log.WithFields(log.Fields{}).Tracef("Producing to topic '%s'", p.TopicReceiptEvent) + _, err := p.JetStream.PublishMsg(m, nats.Context(ctx)) + return err +} + +func (p *SyncAPIProducer) SendToDevice( + ctx context.Context, sender, userID, deviceID, eventType string, + message interface{}, +) error { + devices := []string{} + _, domain, err := gomatrixserverlib.SplitID('@', userID) + if err != nil { + return err + } + + // If the event is targeted locally then we want to expand the wildcard + // out into individual device IDs so that we can send them to each respective + // device. If the event isn't targeted locally then we can't expand the + // wildcard as we don't know about the remote devices, so instead we leave it + // as-is, so that the federation sender can send it on with the wildcard intact. + if domain == p.ServerName && deviceID == "*" { + var res userapi.QueryDevicesResponse + err = p.UserAPI.QueryDevices(context.TODO(), &userapi.QueryDevicesRequest{ + UserID: userID, + }, &res) + if err != nil { + return err + } + for _, dev := range res.Devices { + devices = append(devices, dev.ID) + } + } else { + devices = append(devices, deviceID) + } + + js, err := json.Marshal(message) + if err != nil { + return err + } + + log.WithFields(log.Fields{ + "user_id": userID, + "num_devices": len(devices), + "type": eventType, + }).Tracef("Producing to topic '%s'", p.TopicSendToDeviceEvent) + for _, device := range devices { + ote := &types.OutputSendToDeviceEvent{ + UserID: userID, + DeviceID: device, + SendToDeviceEvent: gomatrixserverlib.SendToDeviceEvent{ + Sender: sender, + Type: eventType, + Content: js, + }, + } + + eventJSON, err := json.Marshal(ote) + if err != nil { + log.WithError(err).Error("sendToDevice failed json.Marshal") + return err + } + m := &nats.Msg{ + Subject: p.TopicSendToDeviceEvent, + Data: eventJSON, + Header: nats.Header{}, + } + m.Header.Set("sender", sender) + m.Header.Set(jetstream.UserID, userID) + if _, err = p.JetStream.PublishMsg(m, nats.Context(ctx)); err != nil { + log.WithError(err).Error("sendToDevice failed t.Producer.SendMessage") + return err + } + } + return nil +} + +func (p *SyncAPIProducer) SendTyping( + ctx context.Context, userID, roomID string, typing bool, timeoutMS int64, +) error { + m := &nats.Msg{ + Subject: p.TopicTypingEvent, + Header: nats.Header{}, + } + m.Header.Set(jetstream.UserID, userID) + m.Header.Set(jetstream.RoomID, roomID) + m.Header.Set("typing", strconv.FormatBool(typing)) + m.Header.Set("timeout_ms", strconv.Itoa(int(timeoutMS))) + + _, err := p.JetStream.PublishMsg(m, nats.Context(ctx)) + return err +} diff --git a/clientapi/routing/account_data.go b/clientapi/routing/account_data.go index d8e982690..873ffaf5d 100644 --- a/clientapi/routing/account_data.go +++ b/clientapi/routing/account_data.go @@ -23,7 +23,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" - eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/internal/eventutil" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/userapi/api" @@ -146,7 +145,7 @@ type fullyReadEvent struct { // SaveReadMarker implements POST /rooms/{roomId}/read_markers func SaveReadMarker( req *http.Request, - userAPI api.UserInternalAPI, rsAPI roomserverAPI.RoomserverInternalAPI, eduAPI eduserverAPI.EDUServerInputAPI, + userAPI api.UserInternalAPI, rsAPI roomserverAPI.RoomserverInternalAPI, syncProducer *producers.SyncAPIProducer, device *api.Device, roomID string, ) util.JSONResponse { // Verify that the user is a member of this room @@ -192,7 +191,7 @@ func SaveReadMarker( // Handle the read receipt that may be included in the read marker if r.Read != "" { - return SetReceipt(req, eduAPI, device, roomID, "m.read", r.Read) + return SetReceipt(req, syncProducer, device, roomID, "m.read", r.Read) } return util.JSONResponse{ diff --git a/clientapi/routing/receipt.go b/clientapi/routing/receipt.go index fe8fe765d..0f9b1b4ff 100644 --- a/clientapi/routing/receipt.go +++ b/clientapi/routing/receipt.go @@ -19,21 +19,20 @@ import ( "net/http" "time" + "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/dendrite/eduserver/api" - userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" "github.com/sirupsen/logrus" ) -func SetReceipt(req *http.Request, eduAPI api.EDUServerInputAPI, device *userapi.Device, roomId, receiptType, eventId string) util.JSONResponse { +func SetReceipt(req *http.Request, syncProducer *producers.SyncAPIProducer, device *userapi.Device, roomID, receiptType, eventID string) util.JSONResponse { timestamp := gomatrixserverlib.AsTimestamp(time.Now()) logrus.WithFields(logrus.Fields{ - "roomId": roomId, + "roomID": roomID, "receiptType": receiptType, - "eventId": eventId, + "eventID": eventID, "userId": device.UserID, "timestamp": timestamp, }).Debug("Setting receipt") @@ -43,7 +42,7 @@ func SetReceipt(req *http.Request, eduAPI api.EDUServerInputAPI, device *userapi return util.MessageResponse(400, fmt.Sprintf("receipt type must be m.read not '%s'", receiptType)) } - if err := api.SendReceipt(req.Context(), eduAPI, device.UserID, roomId, eventId, receiptType, timestamp); err != nil { + if err := syncProducer.SendReceipt(req.Context(), device.UserID, roomID, eventID, receiptType, timestamp); err != nil { return util.ErrorResponse(err) } diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 218087bb3..8afaba560 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -26,7 +26,6 @@ import ( clientutil "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" - eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/transactions" @@ -47,7 +46,6 @@ import ( // nolint: gocyclo func Setup( publicAPIMux, synapseAdminRouter *mux.Router, cfg *config.ClientAPI, - eduAPI eduServerAPI.EDUServerInputAPI, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, userAPI userapi.UserInternalAPI, @@ -467,7 +465,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SendTyping(req, device, vars["roomID"], vars["userID"], eduAPI, rsAPI) + return SendTyping(req, device, vars["roomID"], vars["userID"], rsAPI, syncProducer) }), ).Methods(http.MethodPut, http.MethodOptions) v3mux.Handle("/rooms/{roomID}/redact/{eventID}", @@ -496,7 +494,7 @@ func Setup( return util.ErrorResponse(err) } txnID := vars["txnID"] - return SendToDevice(req, device, eduAPI, transactionsCache, vars["eventType"], &txnID) + return SendToDevice(req, device, syncProducer, transactionsCache, vars["eventType"], &txnID) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -510,7 +508,7 @@ func Setup( return util.ErrorResponse(err) } txnID := vars["txnID"] - return SendToDevice(req, device, eduAPI, transactionsCache, vars["eventType"], &txnID) + return SendToDevice(req, device, syncProducer, transactionsCache, vars["eventType"], &txnID) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -942,7 +940,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SaveReadMarker(req, userAPI, rsAPI, eduAPI, syncProducer, device, vars["roomID"]) + return SaveReadMarker(req, userAPI, rsAPI, syncProducer, device, vars["roomID"]) }), ).Methods(http.MethodPost, http.MethodOptions) @@ -1297,7 +1295,7 @@ func Setup( return util.ErrorResponse(err) } - return SetReceipt(req, eduAPI, device, vars["roomId"], vars["receiptType"], vars["eventId"]) + return SetReceipt(req, syncProducer, device, vars["roomId"], vars["receiptType"], vars["eventId"]) }), ).Methods(http.MethodPost, http.MethodOptions) } diff --git a/clientapi/routing/sendtodevice.go b/clientapi/routing/sendtodevice.go index 768e8e0e7..4a5f08883 100644 --- a/clientapi/routing/sendtodevice.go +++ b/clientapi/routing/sendtodevice.go @@ -18,17 +18,17 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/eduserver/api" + "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/internal/transactions" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" ) // SendToDevice handles PUT /_matrix/client/r0/sendToDevice/{eventType}/{txnId} -// sends the device events to the EDU Server +// sends the device events to the syncapi & federationsender func SendToDevice( req *http.Request, device *userapi.Device, - eduAPI api.EDUServerInputAPI, + syncProducer *producers.SyncAPIProducer, txnCache *transactions.Cache, eventType string, txnID *string, ) util.JSONResponse { @@ -48,8 +48,8 @@ func SendToDevice( for userID, byUser := range httpReq.Messages { for deviceID, message := range byUser { - if err := api.SendToDevice( - req.Context(), eduAPI, device.UserID, userID, deviceID, eventType, message, + if err := syncProducer.SendToDevice( + req.Context(), device.UserID, userID, deviceID, eventType, message, ); err != nil { util.GetLogger(req.Context()).WithError(err).Error("eduProducer.SendToDevice failed") return jsonerror.InternalServerError() diff --git a/clientapi/routing/sendtyping.go b/clientapi/routing/sendtyping.go index abd2061a8..6a27ee615 100644 --- a/clientapi/routing/sendtyping.go +++ b/clientapi/routing/sendtyping.go @@ -17,7 +17,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/eduserver/api" + "github.com/matrix-org/dendrite/clientapi/producers" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" @@ -32,9 +32,8 @@ type typingContentJSON struct { // sends the typing events to client API typingProducer func SendTyping( req *http.Request, device *userapi.Device, roomID string, - userID string, - eduAPI api.EDUServerInputAPI, - rsAPI roomserverAPI.RoomserverInternalAPI, + userID string, rsAPI roomserverAPI.RoomserverInternalAPI, + syncProducer *producers.SyncAPIProducer, ) util.JSONResponse { if device.UserID != userID { return util.JSONResponse{ @@ -56,9 +55,7 @@ func SendTyping( return *resErr } - if err := api.SendTyping( - req.Context(), eduAPI, userID, roomID, r.Typing, r.Timeout, - ); err != nil { + if err := syncProducer.SendTyping(req.Context(), userID, roomID, r.Typing, r.Timeout); err != nil { util.GetLogger(req.Context()).WithError(err).Error("eduProducer.Send failed") return jsonerror.InternalServerError() } diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 8ce641914..26c8eb85f 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -29,7 +29,6 @@ import ( p2pdisc "github.com/libp2p/go-libp2p/p2p/discovery" "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/embed" - "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/keyserver" @@ -40,8 +39,6 @@ import ( "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/dendrite/eduserver/cache" - "github.com/sirupsen/logrus" _ "github.com/mattn/go-sqlite3" @@ -152,9 +149,6 @@ func main() { userAPI := userapi.NewInternalAPI(&base.Base, accountDB, &cfg.UserAPI, nil, keyAPI, rsAPI, base.Base.PushGatewayHTTPClient()) keyAPI.SetUserAPI(userAPI) - eduInputAPI := eduserver.NewInternalAPI( - &base.Base, cache.New(), userAPI, - ) asAPI := appservice.NewInternalAPI(&base.Base, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) fsAPI := federationapi.NewInternalAPI( @@ -180,7 +174,6 @@ func main() { KeyRing: keyRing, AppserviceAPI: asAPI, - EDUInternalAPI: eduInputAPI, FederationAPI: fsAPI, RoomserverAPI: rsAPI, UserAPI: userAPI, diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index 87054dc82..a6abf06e5 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -37,8 +37,6 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms" "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/users" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" - "github.com/matrix-org/dendrite/eduserver" - "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal" @@ -191,10 +189,6 @@ func main() { userAPI := userapi.NewInternalAPI(base, accountDB, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient()) keyAPI.SetUserAPI(userAPI) - eduInputAPI := eduserver.NewInternalAPI( - base, cache.New(), userAPI, - ) - asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) rsComponent.SetFederationAPI(fsAPI, keyRing) @@ -210,7 +204,6 @@ func main() { KeyRing: keyRing, AppserviceAPI: asAPI, - EDUInternalAPI: eduInputAPI, FederationAPI: fsAPI, RoomserverAPI: rsAPI, UserAPI: userAPI, diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index b7e30ba2e..b840eb2b8 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -32,8 +32,6 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggrooms" - "github.com/matrix-org/dendrite/eduserver" - "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal" @@ -120,10 +118,6 @@ func main() { userAPI := userapi.NewInternalAPI(base, accountDB, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient()) keyAPI.SetUserAPI(userAPI) - eduInputAPI := eduserver.NewInternalAPI( - base, cache.New(), userAPI, - ) - asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) fsAPI := federationapi.NewInternalAPI( @@ -139,12 +133,11 @@ func main() { FedClient: federation, KeyRing: keyRing, - AppserviceAPI: asAPI, - EDUInternalAPI: eduInputAPI, - FederationAPI: fsAPI, - RoomserverAPI: rsAPI, - UserAPI: userAPI, - KeyAPI: keyAPI, + AppserviceAPI: asAPI, + FederationAPI: fsAPI, + RoomserverAPI: rsAPI, + UserAPI: userAPI, + KeyAPI: keyAPI, ExtPublicRoomsProvider: yggrooms.NewYggdrasilRoomProvider( ygg, fsAPI, federation, ), diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 3b952504b..1443ab5b1 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -19,8 +19,6 @@ import ( "os" "github.com/matrix-org/dendrite/appservice" - "github.com/matrix-org/dendrite/eduserver" - "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/roomserver" @@ -61,7 +59,6 @@ func main() { // itself. cfg.AppServiceAPI.InternalAPI.Connect = httpAPIAddr cfg.ClientAPI.InternalAPI.Connect = httpAPIAddr - cfg.EDUServer.InternalAPI.Connect = httpAPIAddr cfg.FederationAPI.InternalAPI.Connect = httpAPIAddr cfg.KeyServer.InternalAPI.Connect = httpAPIAddr cfg.MediaAPI.InternalAPI.Connect = httpAPIAddr @@ -136,14 +133,6 @@ func main() { rsImpl.SetUserAPI(userAPI) keyImpl.SetUserAPI(userAPI) - eduInputAPI := eduserver.NewInternalAPI( - base, cache.New(), userAPI, - ) - if base.UseHTTPAPIs { - eduserver.AddInternalRoutes(base.InternalAPIMux, eduInputAPI) - eduInputAPI = base.EDUServerClient() - } - monolith := setup.Monolith{ Config: base.Cfg, AccountDB: accountDB, @@ -151,12 +140,10 @@ func main() { FedClient: federation, KeyRing: keyRing, - AppserviceAPI: asAPI, - EDUInternalAPI: eduInputAPI, - FederationAPI: fsAPI, - RoomserverAPI: rsAPI, - UserAPI: userAPI, - KeyAPI: keyAPI, + AppserviceAPI: asAPI, FederationAPI: fsAPI, + RoomserverAPI: rsAPI, + UserAPI: userAPI, + KeyAPI: keyAPI, } monolith.AddAllPublicRoutes( base.ProcessContext, diff --git a/cmd/dendrite-polylith-multi/main.go b/cmd/dendrite-polylith-multi/main.go index edfe6cdb0..6226cc328 100644 --- a/cmd/dendrite-polylith-multi/main.go +++ b/cmd/dendrite-polylith-multi/main.go @@ -43,7 +43,6 @@ func main() { components := map[string]entrypoint{ "appservice": personalities.Appservice, "clientapi": personalities.ClientAPI, - "eduserver": personalities.EDUServer, "federationapi": personalities.FederationAPI, "keyserver": personalities.KeyServer, "mediaapi": personalities.MediaAPI, diff --git a/cmd/dendrite-polylith-multi/personalities/clientapi.go b/cmd/dendrite-polylith-multi/personalities/clientapi.go index 978d8b0a4..1e509f88a 100644 --- a/cmd/dendrite-polylith-multi/personalities/clientapi.go +++ b/cmd/dendrite-polylith-multi/personalities/clientapi.go @@ -27,13 +27,12 @@ func ClientAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { asQuery := base.AppserviceHTTPClient() rsAPI := base.RoomserverHTTPClient() fsAPI := base.FederationAPIHTTPClient() - eduInputAPI := base.EDUServerClient() userAPI := base.UserAPIClient() keyAPI := base.KeyServerHTTPClient() clientapi.AddPublicRoutes( base.ProcessContext, base.PublicClientAPIMux, base.SynapseAdminMux, &base.Cfg.ClientAPI, - federation, rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, userAPI, userAPI, + federation, rsAPI, asQuery, transactions.New(), fsAPI, userAPI, userAPI, keyAPI, nil, &cfg.MSCs, ) diff --git a/cmd/dendrite-polylith-multi/personalities/eduserver.go b/cmd/dendrite-polylith-multi/personalities/eduserver.go deleted file mode 100644 index 8719facb3..000000000 --- a/cmd/dendrite-polylith-multi/personalities/eduserver.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package personalities - -import ( - "github.com/matrix-org/dendrite/eduserver" - "github.com/matrix-org/dendrite/eduserver/cache" - basepkg "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" -) - -func EDUServer(base *basepkg.BaseDendrite, cfg *config.Dendrite) { - intAPI := eduserver.NewInternalAPI(base, cache.New(), base.UserAPIClient()) - eduserver.AddInternalRoutes(base.InternalAPIMux, intAPI) - - base.SetupAndServeHTTP( - base.Cfg.EDUServer.InternalAPI.Listen, // internal listener - basepkg.NoListener, // external listener - nil, nil, - ) -} diff --git a/cmd/dendrite-polylith-multi/personalities/federationapi.go b/cmd/dendrite-polylith-multi/personalities/federationapi.go index 44357d660..b82577ce3 100644 --- a/cmd/dendrite-polylith-multi/personalities/federationapi.go +++ b/cmd/dendrite-polylith-multi/personalities/federationapi.go @@ -29,9 +29,9 @@ func FederationAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { keyRing := fsAPI.KeyRing() federationapi.AddPublicRoutes( - base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicWellKnownAPIMux, + base.ProcessContext, base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicWellKnownAPIMux, &base.Cfg.FederationAPI, userAPI, federation, keyRing, - rsAPI, fsAPI, base.EDUServerClient(), keyAPI, + rsAPI, fsAPI, keyAPI, &base.Cfg.MSCs, nil, ) diff --git a/cmd/dendritejs-pinecone/main.go b/cmd/dendritejs-pinecone/main.go index 407081f59..05cdf29ee 100644 --- a/cmd/dendritejs-pinecone/main.go +++ b/cmd/dendritejs-pinecone/main.go @@ -31,8 +31,6 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/conn" "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" - "github.com/matrix-org/dendrite/eduserver" - "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/keyserver" @@ -193,7 +191,6 @@ func startup() { userAPI := userapi.NewInternalAPI(base, accountDB, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient()) keyAPI.SetUserAPI(userAPI) - eduInputAPI := eduserver.NewInternalAPI(base, cache.New(), userAPI) asQuery := appservice.NewInternalAPI( base, userAPI, rsAPI, ) @@ -208,12 +205,11 @@ func startup() { FedClient: federation, KeyRing: keyRing, - AppserviceAPI: asQuery, - EDUInternalAPI: eduInputAPI, - FederationAPI: fedSenderAPI, - RoomserverAPI: rsAPI, - UserAPI: userAPI, - KeyAPI: keyAPI, + AppserviceAPI: asQuery, + FederationAPI: fedSenderAPI, + RoomserverAPI: rsAPI, + UserAPI: userAPI, + KeyAPI: keyAPI, //ServerKeyAPI: serverKeyAPI, ExtPublicRoomsProvider: rooms.NewPineconeRoomProvider(pRouter, pSessions, fedSenderAPI, federation), } diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 37cbb12dd..05e0f0ad9 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -24,8 +24,6 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/appservice" - "github.com/matrix-org/dendrite/eduserver" - "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/keyserver" @@ -203,7 +201,6 @@ func main() { } rsAPI := roomserver.NewInternalAPI(base) - eduInputAPI := eduserver.NewInternalAPI(base, cache.New(), userAPI) asQuery := appservice.NewInternalAPI( base, userAPI, rsAPI, ) @@ -222,7 +219,6 @@ func main() { KeyRing: &keyRing, AppserviceAPI: asQuery, - EDUInternalAPI: eduInputAPI, FederationSenderAPI: fedSenderAPI, RoomserverAPI: rsAPI, UserAPI: userAPI, diff --git a/dendrite-config.yaml b/dendrite-config.yaml index 0236851c4..6e2bc7be9 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -187,12 +187,6 @@ client_api: threshold: 5 cooloff_ms: 500 -# Configuration for the EDU server. -edu_server: - internal_api: - listen: http://localhost:7778 # Only used in polylith deployments - connect: http://localhost:7778 # Only used in polylith deployments - # Configuration for the Federation API. federation_api: internal_api: diff --git a/docs/INSTALL.md b/docs/INSTALL.md index a7b2e67f2..523c5c7d1 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -263,14 +263,6 @@ This manages end-to-end encryption keys for users. ./bin/dendrite-polylith-multi --config=dendrite.yaml keyserver ``` -#### EDU server - -This manages processing EDUs such as typing, send-to-device events and presence. Clients do not talk to - -```bash -./bin/dendrite-polylith-multi --config=dendrite.yaml eduserver -``` - #### User server This manages user accounts, device access tokens and user account data, diff --git a/eduserver/api/input.go b/eduserver/api/input.go deleted file mode 100644 index 2aab107b2..000000000 --- a/eduserver/api/input.go +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package api provides the types that are used to communicate with the typing server. -package api - -import ( - "context" - - "github.com/matrix-org/gomatrixserverlib" -) - -// InputTypingEvent is an event for notifying the typing server about typing updates. -type InputTypingEvent struct { - // UserID of the user to update typing status. - UserID string `json:"user_id"` - // RoomID of the room the user is typing (or has stopped). - RoomID string `json:"room_id"` - // Typing is true if the user is typing, false if they have stopped. - Typing bool `json:"typing"` - // Timeout is the interval in milliseconds for which the user should be marked as typing. - TimeoutMS int64 `json:"timeout"` - // OriginServerTS when the server received the update. - OriginServerTS gomatrixserverlib.Timestamp `json:"origin_server_ts"` -} - -type InputSendToDeviceEvent struct { - UserID string `json:"user_id"` - DeviceID string `json:"device_id"` - gomatrixserverlib.SendToDeviceEvent -} - -// InputTypingEventRequest is a request to EDUServerInputAPI -type InputTypingEventRequest struct { - InputTypingEvent InputTypingEvent `json:"input_typing_event"` -} - -// InputTypingEventResponse is a response to InputTypingEvents -type InputTypingEventResponse struct{} - -// InputSendToDeviceEventRequest is a request to EDUServerInputAPI -type InputSendToDeviceEventRequest struct { - InputSendToDeviceEvent InputSendToDeviceEvent `json:"input_send_to_device_event"` -} - -// InputSendToDeviceEventResponse is a response to InputSendToDeviceEventRequest -type InputSendToDeviceEventResponse struct{} - -type InputReceiptEvent struct { - UserID string `json:"user_id"` - RoomID string `json:"room_id"` - EventID string `json:"event_id"` - Type string `json:"type"` - Timestamp gomatrixserverlib.Timestamp `json:"timestamp"` -} - -// InputReceiptEventRequest is a request to EDUServerInputAPI -type InputReceiptEventRequest struct { - InputReceiptEvent InputReceiptEvent `json:"input_receipt_event"` -} - -// InputReceiptEventResponse is a response to InputReceiptEventRequest -type InputReceiptEventResponse struct{} - -type InputCrossSigningKeyUpdateRequest struct { - CrossSigningKeyUpdate `json:"signing_keys"` -} - -type InputCrossSigningKeyUpdateResponse struct{} - -// EDUServerInputAPI is used to write events to the typing server. -type EDUServerInputAPI interface { - InputTypingEvent( - ctx context.Context, - request *InputTypingEventRequest, - response *InputTypingEventResponse, - ) error - - InputSendToDeviceEvent( - ctx context.Context, - request *InputSendToDeviceEventRequest, - response *InputSendToDeviceEventResponse, - ) error - - InputReceiptEvent( - ctx context.Context, - request *InputReceiptEventRequest, - response *InputReceiptEventResponse, - ) error -} diff --git a/eduserver/api/output.go b/eduserver/api/output.go deleted file mode 100644 index c6de4e01c..000000000 --- a/eduserver/api/output.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "time" - - "github.com/matrix-org/gomatrixserverlib" -) - -// OutputTypingEvent is an entry in typing server output kafka log. -// This contains the event with extra fields used to create 'm.typing' event -// in clientapi & federation. -type OutputTypingEvent struct { - // The Event for the typing edu event. - Event TypingEvent `json:"event"` - // ExpireTime is the interval after which the user should no longer be - // considered typing. Only available if Event.Typing is true. - ExpireTime *time.Time -} - -// OutputSendToDeviceEvent is an entry in the send-to-device output kafka log. -// This contains the full event content, along with the user ID and device ID -// to which it is destined. -type OutputSendToDeviceEvent struct { - UserID string `json:"user_id"` - DeviceID string `json:"device_id"` - gomatrixserverlib.SendToDeviceEvent -} - -// OutputReceiptEvent is an entry in the receipt output kafka log -type OutputReceiptEvent struct { - UserID string `json:"user_id"` - RoomID string `json:"room_id"` - EventID string `json:"event_id"` - Type string `json:"type"` - Timestamp gomatrixserverlib.Timestamp `json:"timestamp"` -} - -// OutputCrossSigningKeyUpdate is an entry in the signing key update output kafka log -type OutputCrossSigningKeyUpdate struct { - CrossSigningKeyUpdate `json:"signing_keys"` -} diff --git a/eduserver/api/types.go b/eduserver/api/types.go deleted file mode 100644 index a207580f9..000000000 --- a/eduserver/api/types.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2021 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import "github.com/matrix-org/gomatrixserverlib" - -const ( - MSigningKeyUpdate = "m.signing_key_update" -) - -type TypingEvent struct { - Type string `json:"type"` - RoomID string `json:"room_id"` - UserID string `json:"user_id"` - Typing bool `json:"typing"` -} - -type ReceiptEvent struct { - UserID string `json:"user_id"` - RoomID string `json:"room_id"` - EventID string `json:"event_id"` - Type string `json:"type"` - Timestamp gomatrixserverlib.Timestamp `json:"timestamp"` -} - -type FederationReceiptMRead struct { - User map[string]FederationReceiptData `json:"m.read"` -} - -type FederationReceiptData struct { - Data ReceiptTS `json:"data"` - EventIDs []string `json:"event_ids"` -} - -type ReceiptMRead struct { - User map[string]ReceiptTS `json:"m.read"` -} - -type ReceiptTS struct { - TS gomatrixserverlib.Timestamp `json:"ts"` -} - -type CrossSigningKeyUpdate struct { - MasterKey *gomatrixserverlib.CrossSigningKey `json:"master_key,omitempty"` - SelfSigningKey *gomatrixserverlib.CrossSigningKey `json:"self_signing_key,omitempty"` - UserID string `json:"user_id"` -} diff --git a/eduserver/api/wrapper.go b/eduserver/api/wrapper.go deleted file mode 100644 index 7907f4d39..000000000 --- a/eduserver/api/wrapper.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "context" - "encoding/json" - "time" - - "github.com/matrix-org/gomatrixserverlib" -) - -// SendTyping sends a typing event to EDU server -func SendTyping( - ctx context.Context, eduAPI EDUServerInputAPI, userID, roomID string, - typing bool, timeoutMS int64, -) error { - requestData := InputTypingEvent{ - UserID: userID, - RoomID: roomID, - Typing: typing, - TimeoutMS: timeoutMS, - OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()), - } - - var response InputTypingEventResponse - err := eduAPI.InputTypingEvent( - ctx, &InputTypingEventRequest{InputTypingEvent: requestData}, &response, - ) - - return err -} - -// SendToDevice sends a typing event to EDU server -func SendToDevice( - ctx context.Context, eduAPI EDUServerInputAPI, sender, userID, deviceID, eventType string, - message interface{}, -) error { - js, err := json.Marshal(message) - if err != nil { - return err - } - requestData := InputSendToDeviceEvent{ - UserID: userID, - DeviceID: deviceID, - SendToDeviceEvent: gomatrixserverlib.SendToDeviceEvent{ - Sender: sender, - Type: eventType, - Content: js, - }, - } - request := InputSendToDeviceEventRequest{ - InputSendToDeviceEvent: requestData, - } - response := InputSendToDeviceEventResponse{} - return eduAPI.InputSendToDeviceEvent(ctx, &request, &response) -} - -// SendReceipt sends a receipt event to EDU Server -func SendReceipt( - ctx context.Context, - eduAPI EDUServerInputAPI, userID, roomID, eventID, receiptType string, - timestamp gomatrixserverlib.Timestamp, -) error { - request := InputReceiptEventRequest{ - InputReceiptEvent: InputReceiptEvent{ - UserID: userID, - RoomID: roomID, - EventID: eventID, - Type: receiptType, - Timestamp: timestamp, - }, - } - response := InputReceiptEventResponse{} - return eduAPI.InputReceiptEvent(ctx, &request, &response) -} diff --git a/eduserver/eduserver.go b/eduserver/eduserver.go deleted file mode 100644 index 91208a400..000000000 --- a/eduserver/eduserver.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package eduserver - -import ( - "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/eduserver/api" - "github.com/matrix-org/dendrite/eduserver/cache" - "github.com/matrix-org/dendrite/eduserver/input" - "github.com/matrix-org/dendrite/eduserver/inthttp" - "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/jetstream" - userapi "github.com/matrix-org/dendrite/userapi/api" -) - -// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions -// on the given input API. -func AddInternalRoutes(internalMux *mux.Router, inputAPI api.EDUServerInputAPI) { - inthttp.AddRoutes(inputAPI, internalMux) -} - -// NewInternalAPI returns a concerete implementation of the internal API. Callers -// can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. -func NewInternalAPI( - base *base.BaseDendrite, - eduCache *cache.EDUCache, - userAPI userapi.UserInternalAPI, -) api.EDUServerInputAPI { - cfg := &base.Cfg.EDUServer - - js, _ := jetstream.Prepare(base.ProcessContext, &cfg.Matrix.JetStream) - - return &input.EDUServerInputAPI{ - Cache: eduCache, - UserAPI: userAPI, - JetStream: js, - OutputTypingEventTopic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent), - OutputSendToDeviceEventTopic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), - OutputReceiptEventTopic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent), - ServerName: cfg.Matrix.ServerName, - } -} diff --git a/eduserver/input/input.go b/eduserver/input/input.go deleted file mode 100644 index e58f0dd34..000000000 --- a/eduserver/input/input.go +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package input - -import ( - "context" - "encoding/json" - "time" - - "github.com/matrix-org/dendrite/eduserver/api" - "github.com/matrix-org/dendrite/eduserver/cache" - userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" - "github.com/nats-io/nats.go" - "github.com/sirupsen/logrus" -) - -// EDUServerInputAPI implements api.EDUServerInputAPI -type EDUServerInputAPI struct { - // Cache to store the current typing members in each room. - Cache *cache.EDUCache - // The kafka topic to output new typing events to. - OutputTypingEventTopic string - // The kafka topic to output new send to device events to. - OutputSendToDeviceEventTopic string - // The kafka topic to output new receipt events to - OutputReceiptEventTopic string - // kafka producer - JetStream nats.JetStreamContext - // Internal user query API - UserAPI userapi.UserInternalAPI - // our server name - ServerName gomatrixserverlib.ServerName -} - -// InputTypingEvent implements api.EDUServerInputAPI -func (t *EDUServerInputAPI) InputTypingEvent( - ctx context.Context, - request *api.InputTypingEventRequest, - response *api.InputTypingEventResponse, -) error { - ite := &request.InputTypingEvent - if ite.Typing { - // user is typing, update our current state of users typing. - expireTime := ite.OriginServerTS.Time().Add( - time.Duration(ite.TimeoutMS) * time.Millisecond, - ) - t.Cache.AddTypingUser(ite.UserID, ite.RoomID, &expireTime) - } else { - t.Cache.RemoveUser(ite.UserID, ite.RoomID) - } - - return t.sendTypingEvent(ite) -} - -// InputTypingEvent implements api.EDUServerInputAPI -func (t *EDUServerInputAPI) InputSendToDeviceEvent( - ctx context.Context, - request *api.InputSendToDeviceEventRequest, - response *api.InputSendToDeviceEventResponse, -) error { - ise := &request.InputSendToDeviceEvent - return t.sendToDeviceEvent(ise) -} - -func (t *EDUServerInputAPI) sendTypingEvent(ite *api.InputTypingEvent) error { - ev := &api.TypingEvent{ - Type: gomatrixserverlib.MTyping, - RoomID: ite.RoomID, - UserID: ite.UserID, - Typing: ite.Typing, - } - ote := &api.OutputTypingEvent{ - Event: *ev, - } - - if ev.Typing { - expireTime := ite.OriginServerTS.Time().Add( - time.Duration(ite.TimeoutMS) * time.Millisecond, - ) - ote.ExpireTime = &expireTime - } - - eventJSON, err := json.Marshal(ote) - if err != nil { - return err - } - logrus.WithFields(logrus.Fields{ - "room_id": ite.RoomID, - "user_id": ite.UserID, - "typing": ite.Typing, - }).Tracef("Producing to topic '%s'", t.OutputTypingEventTopic) - - _, err = t.JetStream.PublishMsg(&nats.Msg{ - Subject: t.OutputTypingEventTopic, - Header: nats.Header{}, - Data: eventJSON, - }) - return err -} - -func (t *EDUServerInputAPI) sendToDeviceEvent(ise *api.InputSendToDeviceEvent) error { - devices := []string{} - _, domain, err := gomatrixserverlib.SplitID('@', ise.UserID) - if err != nil { - return err - } - - // If the event is targeted locally then we want to expand the wildcard - // out into individual device IDs so that we can send them to each respective - // device. If the event isn't targeted locally then we can't expand the - // wildcard as we don't know about the remote devices, so instead we leave it - // as-is, so that the federation sender can send it on with the wildcard intact. - if domain == t.ServerName && ise.DeviceID == "*" { - var res userapi.QueryDevicesResponse - err = t.UserAPI.QueryDevices(context.TODO(), &userapi.QueryDevicesRequest{ - UserID: ise.UserID, - }, &res) - if err != nil { - return err - } - for _, dev := range res.Devices { - devices = append(devices, dev.ID) - } - } else { - devices = append(devices, ise.DeviceID) - } - - logrus.WithFields(logrus.Fields{ - "user_id": ise.UserID, - "num_devices": len(devices), - "type": ise.Type, - }).Tracef("Producing to topic '%s'", t.OutputSendToDeviceEventTopic) - for _, device := range devices { - ote := &api.OutputSendToDeviceEvent{ - UserID: ise.UserID, - DeviceID: device, - SendToDeviceEvent: ise.SendToDeviceEvent, - } - - eventJSON, err := json.Marshal(ote) - if err != nil { - logrus.WithError(err).Error("sendToDevice failed json.Marshal") - return err - } - - if _, err = t.JetStream.PublishMsg(&nats.Msg{ - Subject: t.OutputSendToDeviceEventTopic, - Data: eventJSON, - }); err != nil { - logrus.WithError(err).Error("sendToDevice failed t.Producer.SendMessage") - return err - } - } - - return nil -} - -// InputReceiptEvent implements api.EDUServerInputAPI -// TODO: Intelligently batch requests sent by the same user (e.g wait a few milliseconds before emitting output events) -func (t *EDUServerInputAPI) InputReceiptEvent( - ctx context.Context, - request *api.InputReceiptEventRequest, - response *api.InputReceiptEventResponse, -) error { - logrus.WithFields(logrus.Fields{}).Tracef("Producing to topic '%s'", t.OutputReceiptEventTopic) - output := &api.OutputReceiptEvent{ - UserID: request.InputReceiptEvent.UserID, - RoomID: request.InputReceiptEvent.RoomID, - EventID: request.InputReceiptEvent.EventID, - Type: request.InputReceiptEvent.Type, - Timestamp: request.InputReceiptEvent.Timestamp, - } - js, err := json.Marshal(output) - if err != nil { - return err - } - - _, err = t.JetStream.PublishMsg(&nats.Msg{ - Subject: t.OutputReceiptEventTopic, - Data: js, - }) - return err -} diff --git a/eduserver/inthttp/client.go b/eduserver/inthttp/client.go deleted file mode 100644 index 0690ed827..000000000 --- a/eduserver/inthttp/client.go +++ /dev/null @@ -1,70 +0,0 @@ -package inthttp - -import ( - "context" - "errors" - "net/http" - - "github.com/matrix-org/dendrite/eduserver/api" - "github.com/matrix-org/dendrite/internal/httputil" - "github.com/opentracing/opentracing-go" -) - -// HTTP paths for the internal HTTP APIs -const ( - EDUServerInputTypingEventPath = "/eduserver/input" - EDUServerInputSendToDeviceEventPath = "/eduserver/sendToDevice" - EDUServerInputReceiptEventPath = "/eduserver/receipt" -) - -// NewEDUServerClient creates a EDUServerInputAPI implemented by talking to a HTTP POST API. -func NewEDUServerClient(eduServerURL string, httpClient *http.Client) (api.EDUServerInputAPI, error) { - if httpClient == nil { - return nil, errors.New("NewEDUServerClient: httpClient is ") - } - return &httpEDUServerInputAPI{eduServerURL, httpClient}, nil -} - -type httpEDUServerInputAPI struct { - eduServerURL string - httpClient *http.Client -} - -// InputTypingEvent implements EDUServerInputAPI -func (h *httpEDUServerInputAPI) InputTypingEvent( - ctx context.Context, - request *api.InputTypingEventRequest, - response *api.InputTypingEventResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "InputTypingEvent") - defer span.Finish() - - apiURL := h.eduServerURL + EDUServerInputTypingEventPath - return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// InputSendToDeviceEvent implements EDUServerInputAPI -func (h *httpEDUServerInputAPI) InputSendToDeviceEvent( - ctx context.Context, - request *api.InputSendToDeviceEventRequest, - response *api.InputSendToDeviceEventResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "InputSendToDeviceEvent") - defer span.Finish() - - apiURL := h.eduServerURL + EDUServerInputSendToDeviceEventPath - return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// InputSendToDeviceEvent implements EDUServerInputAPI -func (h *httpEDUServerInputAPI) InputReceiptEvent( - ctx context.Context, - request *api.InputReceiptEventRequest, - response *api.InputReceiptEventResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "InputReceiptEventPath") - defer span.Finish() - - apiURL := h.eduServerURL + EDUServerInputReceiptEventPath - return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} diff --git a/eduserver/inthttp/server.go b/eduserver/inthttp/server.go deleted file mode 100644 index a34943750..000000000 --- a/eduserver/inthttp/server.go +++ /dev/null @@ -1,54 +0,0 @@ -package inthttp - -import ( - "encoding/json" - "net/http" - - "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/eduserver/api" - "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/util" -) - -// AddRoutes adds the EDUServerInputAPI handlers to the http.ServeMux. -func AddRoutes(t api.EDUServerInputAPI, internalAPIMux *mux.Router) { - internalAPIMux.Handle(EDUServerInputTypingEventPath, - httputil.MakeInternalAPI("inputTypingEvents", func(req *http.Request) util.JSONResponse { - var request api.InputTypingEventRequest - var response api.InputTypingEventResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.MessageResponse(http.StatusBadRequest, err.Error()) - } - if err := t.InputTypingEvent(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle(EDUServerInputSendToDeviceEventPath, - httputil.MakeInternalAPI("inputSendToDeviceEvents", func(req *http.Request) util.JSONResponse { - var request api.InputSendToDeviceEventRequest - var response api.InputSendToDeviceEventResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.MessageResponse(http.StatusBadRequest, err.Error()) - } - if err := t.InputSendToDeviceEvent(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle(EDUServerInputReceiptEventPath, - httputil.MakeInternalAPI("inputReceiptEvent", func(req *http.Request) util.JSONResponse { - var request api.InputReceiptEventRequest - var response api.InputReceiptEventResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.MessageResponse(http.StatusBadRequest, err.Error()) - } - if err := t.InputReceiptEvent(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) -} diff --git a/federationapi/consumers/eduserver.go b/federationapi/consumers/eduserver.go deleted file mode 100644 index e14e60f47..000000000 --- a/federationapi/consumers/eduserver.go +++ /dev/null @@ -1,257 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package consumers - -import ( - "context" - "encoding/json" - - "github.com/matrix-org/dendrite/eduserver/api" - "github.com/matrix-org/dendrite/federationapi/queue" - "github.com/matrix-org/dendrite/federationapi/storage" - "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/setup/jetstream" - "github.com/matrix-org/dendrite/setup/process" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" - "github.com/nats-io/nats.go" - log "github.com/sirupsen/logrus" -) - -// OutputEDUConsumer consumes events that originate in EDU server. -type OutputEDUConsumer struct { - ctx context.Context - jetstream nats.JetStreamContext - durable string - db storage.Database - queues *queue.OutgoingQueues - ServerName gomatrixserverlib.ServerName - typingTopic string - sendToDeviceTopic string - receiptTopic string -} - -// NewOutputEDUConsumer creates a new OutputEDUConsumer. Call Start() to begin consuming from EDU servers. -func NewOutputEDUConsumer( - process *process.ProcessContext, - cfg *config.FederationAPI, - js nats.JetStreamContext, - queues *queue.OutgoingQueues, - store storage.Database, -) *OutputEDUConsumer { - return &OutputEDUConsumer{ - ctx: process.Context(), - jetstream: js, - queues: queues, - db: store, - ServerName: cfg.Matrix.ServerName, - durable: cfg.Matrix.JetStream.Durable("FederationAPIEDUServerConsumer"), - typingTopic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent), - sendToDeviceTopic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), - receiptTopic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent), - } -} - -// Start consuming from EDU servers -func (t *OutputEDUConsumer) Start() error { - if err := jetstream.JetStreamConsumer( - t.ctx, t.jetstream, t.typingTopic, t.durable, t.onTypingEvent, - nats.DeliverAll(), nats.ManualAck(), - ); err != nil { - return err - } - if err := jetstream.JetStreamConsumer( - t.ctx, t.jetstream, t.sendToDeviceTopic, t.durable, t.onSendToDeviceEvent, - nats.DeliverAll(), nats.ManualAck(), - ); err != nil { - return err - } - if err := jetstream.JetStreamConsumer( - t.ctx, t.jetstream, t.receiptTopic, t.durable, t.onReceiptEvent, - nats.DeliverAll(), nats.ManualAck(), - ); err != nil { - return err - } - return nil -} - -// onSendToDeviceEvent is called in response to a message received on the -// send-to-device events topic from the EDU server. -func (t *OutputEDUConsumer) onSendToDeviceEvent(ctx context.Context, msg *nats.Msg) bool { - // Extract the send-to-device event from msg. - var ote api.OutputSendToDeviceEvent - if err := json.Unmarshal(msg.Data, &ote); err != nil { - log.WithError(err).Errorf("eduserver output log: message parse failed (expected send-to-device)") - return true - } - - // only send send-to-device events which originated from us - _, originServerName, err := gomatrixserverlib.SplitID('@', ote.Sender) - if err != nil { - log.WithError(err).WithField("user_id", ote.Sender).Error("Failed to extract domain from send-to-device sender") - return true - } - if originServerName != t.ServerName { - log.WithField("other_server", originServerName).Info("Suppressing send-to-device: originated elsewhere") - return true - } - - _, destServerName, err := gomatrixserverlib.SplitID('@', ote.UserID) - if err != nil { - log.WithError(err).WithField("user_id", ote.UserID).Error("Failed to extract domain from send-to-device destination") - return true - } - - // Pack the EDU and marshal it - edu := &gomatrixserverlib.EDU{ - Type: gomatrixserverlib.MDirectToDevice, - Origin: string(t.ServerName), - } - tdm := gomatrixserverlib.ToDeviceMessage{ - Sender: ote.Sender, - Type: ote.Type, - MessageID: util.RandomString(32), - Messages: map[string]map[string]json.RawMessage{ - ote.UserID: { - ote.DeviceID: ote.Content, - }, - }, - } - if edu.Content, err = json.Marshal(tdm); err != nil { - log.WithError(err).Error("failed to marshal EDU JSON") - return true - } - - log.Debugf("Sending send-to-device message into %q destination queue", destServerName) - if err := t.queues.SendEDU(edu, t.ServerName, []gomatrixserverlib.ServerName{destServerName}); err != nil { - log.WithError(err).Error("failed to send EDU") - return false - } - - return true -} - -// onTypingEvent is called in response to a message received on the typing -// events topic from the EDU server. -func (t *OutputEDUConsumer) onTypingEvent(ctx context.Context, msg *nats.Msg) bool { - // Extract the typing event from msg. - var ote api.OutputTypingEvent - if err := json.Unmarshal(msg.Data, &ote); err != nil { - // Skip this msg but continue processing messages. - log.WithError(err).Errorf("eduserver output log: message parse failed (expected typing)") - _ = msg.Ack() - return true - } - - // only send typing events which originated from us - _, typingServerName, err := gomatrixserverlib.SplitID('@', ote.Event.UserID) - if err != nil { - log.WithError(err).WithField("user_id", ote.Event.UserID).Error("Failed to extract domain from typing sender") - _ = msg.Ack() - return true - } - if typingServerName != t.ServerName { - return true - } - - joined, err := t.db.GetJoinedHosts(ctx, ote.Event.RoomID) - if err != nil { - log.WithError(err).WithField("room_id", ote.Event.RoomID).Error("failed to get joined hosts for room") - return false - } - - names := make([]gomatrixserverlib.ServerName, len(joined)) - for i := range joined { - names[i] = joined[i].ServerName - } - - edu := &gomatrixserverlib.EDU{Type: ote.Event.Type} - if edu.Content, err = json.Marshal(map[string]interface{}{ - "room_id": ote.Event.RoomID, - "user_id": ote.Event.UserID, - "typing": ote.Event.Typing, - }); err != nil { - log.WithError(err).Error("failed to marshal EDU JSON") - return true - } - - if err := t.queues.SendEDU(edu, t.ServerName, names); err != nil { - log.WithError(err).Error("failed to send EDU") - return false - } - - return true -} - -// onReceiptEvent is called in response to a message received on the receipt -// events topic from the EDU server. -func (t *OutputEDUConsumer) onReceiptEvent(ctx context.Context, msg *nats.Msg) bool { - // Extract the typing event from msg. - var receipt api.OutputReceiptEvent - if err := json.Unmarshal(msg.Data, &receipt); err != nil { - // Skip this msg but continue processing messages. - log.WithError(err).Errorf("eduserver output log: message parse failed (expected receipt)") - return true - } - - // only send receipt events which originated from us - _, receiptServerName, err := gomatrixserverlib.SplitID('@', receipt.UserID) - if err != nil { - log.WithError(err).WithField("user_id", receipt.UserID).Error("failed to extract domain from receipt sender") - return true - } - if receiptServerName != t.ServerName { - return true - } - - joined, err := t.db.GetJoinedHosts(ctx, receipt.RoomID) - if err != nil { - log.WithError(err).WithField("room_id", receipt.RoomID).Error("failed to get joined hosts for room") - return false - } - - names := make([]gomatrixserverlib.ServerName, len(joined)) - for i := range joined { - names[i] = joined[i].ServerName - } - - content := map[string]api.FederationReceiptMRead{} - content[receipt.RoomID] = api.FederationReceiptMRead{ - User: map[string]api.FederationReceiptData{ - receipt.UserID: { - Data: api.ReceiptTS{ - TS: receipt.Timestamp, - }, - EventIDs: []string{receipt.EventID}, - }, - }, - } - - edu := &gomatrixserverlib.EDU{ - Type: gomatrixserverlib.MReceipt, - Origin: string(t.ServerName), - } - if edu.Content, err = json.Marshal(content); err != nil { - log.WithError(err).Error("failed to marshal EDU JSON") - return true - } - - if err := t.queues.SendEDU(edu, t.ServerName, names); err != nil { - log.WithError(err).Error("failed to send EDU") - return false - } - - return true -} diff --git a/federationapi/consumers/keychange.go b/federationapi/consumers/keychange.go index 94e454359..0ece18e97 100644 --- a/federationapi/consumers/keychange.go +++ b/federationapi/consumers/keychange.go @@ -18,9 +18,9 @@ import ( "context" "encoding/json" - eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/federationapi/queue" "github.com/matrix-org/dendrite/federationapi/storage" + "github.com/matrix-org/dendrite/federationapi/types" "github.com/matrix-org/dendrite/keyserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" @@ -190,7 +190,7 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool { // Pack the EDU and marshal it edu := &gomatrixserverlib.EDU{ - Type: eduserverAPI.MSigningKeyUpdate, + Type: types.MSigningKeyUpdate, Origin: string(t.serverName), } if edu.Content, err = json.Marshal(output); err != nil { diff --git a/federationapi/consumers/receipts.go b/federationapi/consumers/receipts.go new file mode 100644 index 000000000..9300451eb --- /dev/null +++ b/federationapi/consumers/receipts.go @@ -0,0 +1,141 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package consumers + +import ( + "context" + "encoding/json" + "strconv" + + "github.com/getsentry/sentry-go" + "github.com/matrix-org/dendrite/federationapi/queue" + "github.com/matrix-org/dendrite/federationapi/storage" + fedTypes "github.com/matrix-org/dendrite/federationapi/types" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" + syncTypes "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/gomatrixserverlib" + "github.com/nats-io/nats.go" + log "github.com/sirupsen/logrus" +) + +// OutputReceiptConsumer consumes events that originate in the clientapi. +type OutputReceiptConsumer struct { + ctx context.Context + jetstream nats.JetStreamContext + durable string + db storage.Database + queues *queue.OutgoingQueues + ServerName gomatrixserverlib.ServerName + topic string +} + +// NewOutputReceiptConsumer creates a new OutputReceiptConsumer. Call Start() to begin consuming typing events. +func NewOutputReceiptConsumer( + process *process.ProcessContext, + cfg *config.FederationAPI, + js nats.JetStreamContext, + queues *queue.OutgoingQueues, + store storage.Database, +) *OutputReceiptConsumer { + return &OutputReceiptConsumer{ + ctx: process.Context(), + jetstream: js, + queues: queues, + db: store, + ServerName: cfg.Matrix.ServerName, + durable: cfg.Matrix.JetStream.Durable("FederationAPIReceiptConsumer"), + topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent), + } +} + +// Start consuming from the clientapi +func (t *OutputReceiptConsumer) Start() error { + return jetstream.JetStreamConsumer( + t.ctx, t.jetstream, t.topic, t.durable, t.onMessage, + nats.DeliverAll(), nats.ManualAck(), nats.HeadersOnly(), + ) +} + +// onMessage is called in response to a message received on the receipt +// events topic from the client api. +func (t *OutputReceiptConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool { + receipt := syncTypes.OutputReceiptEvent{ + UserID: msg.Header.Get(jetstream.UserID), + RoomID: msg.Header.Get(jetstream.RoomID), + EventID: msg.Header.Get(jetstream.EventID), + Type: msg.Header.Get("type"), + } + + // only send receipt events which originated from us + _, receiptServerName, err := gomatrixserverlib.SplitID('@', receipt.UserID) + if err != nil { + log.WithError(err).WithField("user_id", receipt.UserID).Error("failed to extract domain from receipt sender") + return true + } + if receiptServerName != t.ServerName { + return true + } + + timestamp, err := strconv.Atoi(msg.Header.Get("timestamp")) + if err != nil { + // If the message was invalid, log it and move on to the next message in the stream + log.WithError(err).Errorf("EDU output log: message parse failure") + sentry.CaptureException(err) + return true + } + + receipt.Timestamp = gomatrixserverlib.Timestamp(timestamp) + + joined, err := t.db.GetJoinedHosts(ctx, receipt.RoomID) + if err != nil { + log.WithError(err).WithField("room_id", receipt.RoomID).Error("failed to get joined hosts for room") + return false + } + + names := make([]gomatrixserverlib.ServerName, len(joined)) + for i := range joined { + names[i] = joined[i].ServerName + } + + content := map[string]fedTypes.FederationReceiptMRead{} + content[receipt.RoomID] = fedTypes.FederationReceiptMRead{ + User: map[string]fedTypes.FederationReceiptData{ + receipt.UserID: { + Data: fedTypes.ReceiptTS{ + TS: receipt.Timestamp, + }, + EventIDs: []string{receipt.EventID}, + }, + }, + } + + edu := &gomatrixserverlib.EDU{ + Type: gomatrixserverlib.MReceipt, + Origin: string(t.ServerName), + } + if edu.Content, err = json.Marshal(content); err != nil { + log.WithError(err).Error("failed to marshal EDU JSON") + return true + } + + if err := t.queues.SendEDU(edu, t.ServerName, names); err != nil { + log.WithError(err).Error("failed to send EDU") + return false + } + + return true +} diff --git a/federationapi/consumers/sendtodevice.go b/federationapi/consumers/sendtodevice.go new file mode 100644 index 000000000..84c9f620d --- /dev/null +++ b/federationapi/consumers/sendtodevice.go @@ -0,0 +1,125 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package consumers + +import ( + "context" + "encoding/json" + + "github.com/matrix-org/dendrite/federationapi/queue" + "github.com/matrix-org/dendrite/federationapi/storage" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" + syncTypes "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/nats-io/nats.go" + log "github.com/sirupsen/logrus" +) + +// OutputSendToDeviceConsumer consumes events that originate in the clientapi. +type OutputSendToDeviceConsumer struct { + ctx context.Context + jetstream nats.JetStreamContext + durable string + db storage.Database + queues *queue.OutgoingQueues + ServerName gomatrixserverlib.ServerName + topic string +} + +// NewOutputSendToDeviceConsumer creates a new OutputSendToDeviceConsumer. Call Start() to begin consuming send-to-device events. +func NewOutputSendToDeviceConsumer( + process *process.ProcessContext, + cfg *config.FederationAPI, + js nats.JetStreamContext, + queues *queue.OutgoingQueues, + store storage.Database, +) *OutputSendToDeviceConsumer { + return &OutputSendToDeviceConsumer{ + ctx: process.Context(), + jetstream: js, + queues: queues, + db: store, + ServerName: cfg.Matrix.ServerName, + durable: cfg.Matrix.JetStream.Durable("FederationAPIESendToDeviceConsumer"), + topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), + } +} + +// Start consuming from the client api +func (t *OutputSendToDeviceConsumer) Start() error { + return jetstream.JetStreamConsumer( + t.ctx, t.jetstream, t.topic, t.durable, t.onMessage, + nats.DeliverAll(), nats.ManualAck(), + ) +} + +// onMessage is called in response to a message received on the +// send-to-device events topic from the client api. +func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool { + // only send send-to-device events which originated from us + sender := msg.Header.Get("sender") + _, originServerName, err := gomatrixserverlib.SplitID('@', sender) + if err != nil { + log.WithError(err).WithField("user_id", sender).Error("Failed to extract domain from send-to-device sender") + return true + } + if originServerName != t.ServerName { + log.WithField("other_server", originServerName).Info("Suppressing send-to-device: originated elsewhere") + return true + } + // Extract the send-to-device event from msg. + var ote syncTypes.OutputSendToDeviceEvent + if err = json.Unmarshal(msg.Data, &ote); err != nil { + log.WithError(err).Errorf("output log: message parse failed (expected send-to-device)") + return true + } + + _, destServerName, err := gomatrixserverlib.SplitID('@', ote.UserID) + if err != nil { + log.WithError(err).WithField("user_id", ote.UserID).Error("Failed to extract domain from send-to-device destination") + return true + } + + // Pack the EDU and marshal it + edu := &gomatrixserverlib.EDU{ + Type: gomatrixserverlib.MDirectToDevice, + Origin: string(t.ServerName), + } + tdm := gomatrixserverlib.ToDeviceMessage{ + Sender: ote.Sender, + Type: ote.Type, + MessageID: util.RandomString(32), + Messages: map[string]map[string]json.RawMessage{ + ote.UserID: { + ote.DeviceID: ote.Content, + }, + }, + } + if edu.Content, err = json.Marshal(tdm); err != nil { + log.WithError(err).Error("failed to marshal EDU JSON") + return true + } + + log.Debugf("Sending send-to-device message into %q destination queue", destServerName) + if err := t.queues.SendEDU(edu, t.ServerName, []gomatrixserverlib.ServerName{destServerName}); err != nil { + log.WithError(err).Error("failed to send EDU") + return false + } + + return true +} diff --git a/federationapi/consumers/typing.go b/federationapi/consumers/typing.go new file mode 100644 index 000000000..428e1a867 --- /dev/null +++ b/federationapi/consumers/typing.go @@ -0,0 +1,119 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package consumers + +import ( + "context" + "encoding/json" + "strconv" + + "github.com/matrix-org/dendrite/federationapi/queue" + "github.com/matrix-org/dendrite/federationapi/storage" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" + "github.com/matrix-org/gomatrixserverlib" + "github.com/nats-io/nats.go" + log "github.com/sirupsen/logrus" +) + +// OutputTypingConsumer consumes events that originate in the clientapi. +type OutputTypingConsumer struct { + ctx context.Context + jetstream nats.JetStreamContext + durable string + db storage.Database + queues *queue.OutgoingQueues + ServerName gomatrixserverlib.ServerName + topic string +} + +// NewOutputTypingConsumer creates a new OutputTypingConsumer. Call Start() to begin consuming typing events. +func NewOutputTypingConsumer( + process *process.ProcessContext, + cfg *config.FederationAPI, + js nats.JetStreamContext, + queues *queue.OutgoingQueues, + store storage.Database, +) *OutputTypingConsumer { + return &OutputTypingConsumer{ + ctx: process.Context(), + jetstream: js, + queues: queues, + db: store, + ServerName: cfg.Matrix.ServerName, + durable: cfg.Matrix.JetStream.Durable("FederationAPITypingConsumer"), + topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent), + } +} + +// Start consuming from the clientapi +func (t *OutputTypingConsumer) Start() error { + return jetstream.JetStreamConsumer( + t.ctx, t.jetstream, t.topic, t.durable, t.onMessage, + nats.DeliverAll(), nats.ManualAck(), nats.HeadersOnly(), + ) +} + +// onMessage is called in response to a message received on the typing +// events topic from the client api. +func (t *OutputTypingConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool { + // Extract the typing event from msg. + roomID := msg.Header.Get(jetstream.RoomID) + userID := msg.Header.Get(jetstream.UserID) + typing, err := strconv.ParseBool(msg.Header.Get("typing")) + if err != nil { + log.WithError(err).Errorf("EDU output log: typing parse failure") + return true + } + + // only send typing events which originated from us + _, typingServerName, err := gomatrixserverlib.SplitID('@', userID) + if err != nil { + log.WithError(err).WithField("user_id", userID).Error("Failed to extract domain from typing sender") + _ = msg.Ack() + return true + } + if typingServerName != t.ServerName { + return true + } + + joined, err := t.db.GetJoinedHosts(ctx, roomID) + if err != nil { + log.WithError(err).WithField("room_id", roomID).Error("failed to get joined hosts for room") + return false + } + + names := make([]gomatrixserverlib.ServerName, len(joined)) + for i := range joined { + names[i] = joined[i].ServerName + } + + edu := &gomatrixserverlib.EDU{Type: "m.typing"} + if edu.Content, err = json.Marshal(map[string]interface{}{ + "room_id": roomID, + "user_id": userID, + "typing": typing, + }); err != nil { + log.WithError(err).Error("failed to marshal EDU JSON") + return true + } + if err := t.queues.SendEDU(edu, t.ServerName, names); err != nil { + log.WithError(err).Error("failed to send EDU") + return false + } + + return true +} diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index b7f93ecb9..8a0ce8e37 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -16,12 +16,12 @@ package federationapi import ( "github.com/gorilla/mux" - eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/federationapi/api" federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/federationapi/consumers" "github.com/matrix-org/dendrite/federationapi/internal" "github.com/matrix-org/dendrite/federationapi/inthttp" + "github.com/matrix-org/dendrite/federationapi/producers" "github.com/matrix-org/dendrite/federationapi/queue" "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/federationapi/storage" @@ -31,6 +31,7 @@ import ( "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/sirupsen/logrus" @@ -46,6 +47,7 @@ func AddInternalRoutes(router *mux.Router, intAPI api.FederationInternalAPI) { // AddPublicRoutes sets up and registers HTTP handlers on the base API muxes for the FederationAPI component. func AddPublicRoutes( + process *process.ProcessContext, fedRouter, keyRouter, wellKnownRouter *mux.Router, cfg *config.FederationAPI, userAPI userapi.UserInternalAPI, @@ -53,16 +55,26 @@ func AddPublicRoutes( keyRing gomatrixserverlib.JSONVerifier, rsAPI roomserverAPI.RoomserverInternalAPI, federationAPI federationAPI.FederationInternalAPI, - eduAPI eduserverAPI.EDUServerInputAPI, keyAPI keyserverAPI.KeyInternalAPI, mscCfg *config.MSCs, servers federationAPI.ServersInRoomProvider, ) { + + js, _ := jetstream.Prepare(process, &cfg.Matrix.JetStream) + producer := &producers.SyncAPIProducer{ + JetStream: js, + TopicReceiptEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent), + TopicSendToDeviceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), + TopicTypingEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent), + ServerName: cfg.Matrix.ServerName, + UserAPI: userAPI, + } + routing.Setup( fedRouter, keyRouter, wellKnownRouter, cfg, rsAPI, - eduAPI, federationAPI, keyRing, + federationAPI, keyRing, federation, userAPI, keyAPI, mscCfg, - servers, + servers, producer, ) } @@ -112,17 +124,28 @@ func NewInternalAPI( if err = rsConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start room server consumer") } - - tsConsumer := consumers.NewOutputEDUConsumer( + tsConsumer := consumers.NewOutputSendToDeviceConsumer( base.ProcessContext, cfg, js, queues, federationDB, ) - if err := tsConsumer.Start(); err != nil { - logrus.WithError(err).Panic("failed to start typing server consumer") + if err = tsConsumer.Start(); err != nil { + logrus.WithError(err).Panic("failed to start send-to-device consumer") + } + receiptConsumer := consumers.NewOutputReceiptConsumer( + base.ProcessContext, cfg, js, queues, federationDB, + ) + if err = receiptConsumer.Start(); err != nil { + logrus.WithError(err).Panic("failed to start receipt consumer") + } + typingConsumer := consumers.NewOutputTypingConsumer( + base.ProcessContext, cfg, js, queues, federationDB, + ) + if err = typingConsumer.Start(); err != nil { + logrus.WithError(err).Panic("failed to start typing consumer") } keyConsumer := consumers.NewKeyChangeConsumer( base.ProcessContext, &base.Cfg.KeyServer, js, queues, federationDB, rsAPI, ) - if err := keyConsumer.Start(); err != nil { + if err = keyConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start key server consumer") } diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go index c660f12e0..833359c11 100644 --- a/federationapi/federationapi_test.go +++ b/federationapi/federationapi_test.go @@ -30,7 +30,7 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) { fsAPI := base.FederationAPIHTTPClient() // TODO: This is pretty fragile, as if anything calls anything on these nils this test will break. // Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing. - federationapi.AddPublicRoutes(base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicWellKnownAPIMux, &cfg.FederationAPI, nil, nil, keyRing, nil, fsAPI, nil, nil, &cfg.MSCs, nil) + federationapi.AddPublicRoutes(base.ProcessContext, base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicWellKnownAPIMux, &cfg.FederationAPI, nil, nil, keyRing, nil, fsAPI, nil, &cfg.MSCs, nil) baseURL, cancel := test.ListenAndServe(t, base.PublicFederationAPIMux, true) defer cancel() serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://")) diff --git a/federationapi/producers/syncapi.go b/federationapi/producers/syncapi.go new file mode 100644 index 000000000..24acb1268 --- /dev/null +++ b/federationapi/producers/syncapi.go @@ -0,0 +1,144 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package producers + +import ( + "context" + "encoding/json" + "strconv" + + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/syncapi/types" + userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/nats-io/nats.go" + log "github.com/sirupsen/logrus" +) + +// SyncAPIProducer produces events for the sync API server to consume +type SyncAPIProducer struct { + TopicReceiptEvent string + TopicSendToDeviceEvent string + TopicTypingEvent string + JetStream nats.JetStreamContext + ServerName gomatrixserverlib.ServerName + UserAPI userapi.UserInternalAPI +} + +func (p *SyncAPIProducer) SendReceipt( + ctx context.Context, + userID, roomID, eventID, receiptType string, timestamp gomatrixserverlib.Timestamp, +) error { + m := &nats.Msg{ + Subject: p.TopicReceiptEvent, + Header: nats.Header{}, + } + m.Header.Set(jetstream.UserID, userID) + m.Header.Set(jetstream.RoomID, roomID) + m.Header.Set(jetstream.EventID, eventID) + m.Header.Set("type", receiptType) + m.Header.Set("timestamp", strconv.Itoa(int(timestamp))) + + log.WithFields(log.Fields{}).Tracef("Producing to topic '%s'", p.TopicReceiptEvent) + _, err := p.JetStream.PublishMsg(m, nats.Context(ctx)) + return err +} + +func (p *SyncAPIProducer) SendToDevice( + ctx context.Context, sender, userID, deviceID, eventType string, + message interface{}, +) error { + devices := []string{} + _, domain, err := gomatrixserverlib.SplitID('@', userID) + if err != nil { + return err + } + + // If the event is targeted locally then we want to expand the wildcard + // out into individual device IDs so that we can send them to each respective + // device. If the event isn't targeted locally then we can't expand the + // wildcard as we don't know about the remote devices, so instead we leave it + // as-is, so that the federation sender can send it on with the wildcard intact. + if domain == p.ServerName && deviceID == "*" { + var res userapi.QueryDevicesResponse + err = p.UserAPI.QueryDevices(context.TODO(), &userapi.QueryDevicesRequest{ + UserID: userID, + }, &res) + if err != nil { + return err + } + for _, dev := range res.Devices { + devices = append(devices, dev.ID) + } + } else { + devices = append(devices, deviceID) + } + + js, err := json.Marshal(message) + if err != nil { + return err + } + + log.WithFields(log.Fields{ + "user_id": userID, + "num_devices": len(devices), + "type": eventType, + }).Tracef("Producing to topic '%s'", p.TopicSendToDeviceEvent) + for _, device := range devices { + ote := &types.OutputSendToDeviceEvent{ + UserID: userID, + DeviceID: device, + SendToDeviceEvent: gomatrixserverlib.SendToDeviceEvent{ + Sender: sender, + Type: eventType, + Content: js, + }, + } + + eventJSON, err := json.Marshal(ote) + if err != nil { + log.WithError(err).Error("sendToDevice failed json.Marshal") + return err + } + m := &nats.Msg{ + Subject: p.TopicSendToDeviceEvent, + Data: eventJSON, + Header: nats.Header{}, + } + m.Header.Set("sender", sender) + m.Header.Set(jetstream.UserID, userID) + + if _, err = p.JetStream.PublishMsg(m, nats.Context(ctx)); err != nil { + log.WithError(err).Error("sendToDevice failed t.Producer.SendMessage") + return err + } + } + return nil +} + +func (p *SyncAPIProducer) SendTyping( + ctx context.Context, userID, roomID string, typing bool, timeoutMS int64, +) error { + m := &nats.Msg{ + Subject: p.TopicTypingEvent, + Header: nats.Header{}, + } + m.Header.Set(jetstream.UserID, userID) + m.Header.Set(jetstream.RoomID, roomID) + m.Header.Set("typing", strconv.FormatBool(typing)) + m.Header.Set("timeout_ms", strconv.Itoa(int(timeoutMS))) + _, err := p.JetStream.PublishMsg(m, nats.Context(ctx)) + return err +} diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 04c88d957..9e5cdb28b 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -19,8 +19,8 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/clientapi/jsonerror" - eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" federationAPI "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/federationapi/producers" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/httputil" keyserverAPI "github.com/matrix-org/dendrite/keyserver/api" @@ -44,7 +44,6 @@ func Setup( fedMux, keyMux, wkMux *mux.Router, cfg *config.FederationAPI, rsAPI roomserverAPI.RoomserverInternalAPI, - eduAPI eduserverAPI.EDUServerInputAPI, fsAPI federationAPI.FederationInternalAPI, keys gomatrixserverlib.JSONVerifier, federation *gomatrixserverlib.FederationClient, @@ -52,6 +51,7 @@ func Setup( keyAPI keyserverAPI.KeyInternalAPI, mscCfg *config.MSCs, servers federationAPI.ServersInRoomProvider, + producer *producers.SyncAPIProducer, ) { v2keysmux := keyMux.PathPrefix("/v2").Subrouter() v1fedmux := fedMux.PathPrefix("/v1").Subrouter() @@ -116,7 +116,7 @@ func Setup( func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return Send( httpReq, request, gomatrixserverlib.TransactionID(vars["txnID"]), - cfg, rsAPI, eduAPI, keyAPI, keys, federation, mu, servers, + cfg, rsAPI, keyAPI, keys, federation, mu, servers, producer, ) }, )).Methods(http.MethodPut, http.MethodOptions) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 745e36de9..eacc76db3 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -23,8 +23,9 @@ import ( "time" "github.com/matrix-org/dendrite/clientapi/jsonerror" - eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" federationAPI "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/federationapi/producers" + "github.com/matrix-org/dendrite/federationapi/types" "github.com/matrix-org/dendrite/internal" keyapi "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/roomserver/api" @@ -87,12 +88,12 @@ func Send( txnID gomatrixserverlib.TransactionID, cfg *config.FederationAPI, rsAPI api.RoomserverInternalAPI, - eduAPI eduserverAPI.EDUServerInputAPI, keyAPI keyapi.KeyInternalAPI, keys gomatrixserverlib.JSONVerifier, federation *gomatrixserverlib.FederationClient, mu *internal.MutexByRoom, servers federationAPI.ServersInRoomProvider, + producer *producers.SyncAPIProducer, ) util.JSONResponse { // First we should check if this origin has already submitted this // txn ID to us. If they have and the txnIDs map contains an entry, @@ -127,12 +128,12 @@ func Send( t := txnReq{ rsAPI: rsAPI, - eduAPI: eduAPI, keys: keys, federation: federation, servers: servers, keyAPI: keyAPI, roomsMu: mu, + producer: producer, } var txnEvents struct { @@ -185,12 +186,12 @@ func Send( type txnReq struct { gomatrixserverlib.Transaction rsAPI api.RoomserverInternalAPI - eduAPI eduserverAPI.EDUServerInputAPI keyAPI keyapi.KeyInternalAPI keys gomatrixserverlib.JSONVerifier federation txnFederationClient roomsMu *internal.MutexByRoom servers federationAPI.ServersInRoomProvider + producer *producers.SyncAPIProducer } // A subset of FederationClient functionality that txn requires. Useful for testing. @@ -329,8 +330,8 @@ func (t *txnReq) processEDUs(ctx context.Context) { util.GetLogger(ctx).Debugf("Dropping typing event where sender domain (%q) doesn't match origin (%q)", domain, t.Origin) continue } - if err := eduserverAPI.SendTyping(ctx, t.eduAPI, typingPayload.UserID, typingPayload.RoomID, typingPayload.Typing, 30*1000); err != nil { - util.GetLogger(ctx).WithError(err).Error("Failed to send typing event to edu server") + if err := t.producer.SendTyping(ctx, typingPayload.UserID, typingPayload.RoomID, typingPayload.Typing, 30*1000); err != nil { + util.GetLogger(ctx).WithError(err).Error("Failed to send typing event to JetStream") } case gomatrixserverlib.MDirectToDevice: // https://matrix.org/docs/spec/server_server/r0.1.3#m-direct-to-device-schema @@ -342,12 +343,12 @@ func (t *txnReq) processEDUs(ctx context.Context) { for userID, byUser := range directPayload.Messages { for deviceID, message := range byUser { // TODO: check that the user and the device actually exist here - if err := eduserverAPI.SendToDevice(ctx, t.eduAPI, directPayload.Sender, userID, deviceID, directPayload.Type, message); err != nil { + if err := t.producer.SendToDevice(ctx, directPayload.Sender, userID, deviceID, directPayload.Type, message); err != nil { util.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{ "sender": directPayload.Sender, "user_id": userID, "device_id": deviceID, - }).Error("Failed to send send-to-device event to edu server") + }).Error("Failed to send send-to-device event to JetStream") } } } @@ -355,7 +356,7 @@ func (t *txnReq) processEDUs(ctx context.Context) { t.processDeviceListUpdate(ctx, e) case gomatrixserverlib.MReceipt: // https://matrix.org/docs/spec/server_server/r0.1.4#receipts - payload := map[string]eduserverAPI.FederationReceiptMRead{} + payload := map[string]types.FederationReceiptMRead{} if err := json.Unmarshal(e.Content, &payload); err != nil { util.GetLogger(ctx).WithError(err).Debug("Failed to unmarshal receipt event") @@ -379,12 +380,12 @@ func (t *txnReq) processEDUs(ctx context.Context) { "user_id": userID, "room_id": roomID, "events": mread.EventIDs, - }).Error("Failed to send receipt event to edu server") + }).Error("Failed to send receipt event to JetStream") continue } } } - case eduserverAPI.MSigningKeyUpdate: + case types.MSigningKeyUpdate: if err := t.processSigningKeyUpdate(ctx, e); err != nil { logrus.WithError(err).Errorf("Failed to process signing key update") } @@ -395,7 +396,7 @@ func (t *txnReq) processEDUs(ctx context.Context) { } func (t *txnReq) processSigningKeyUpdate(ctx context.Context, e gomatrixserverlib.EDU) error { - var updatePayload eduserverAPI.CrossSigningKeyUpdate + var updatePayload keyapi.CrossSigningKeyUpdate if err := json.Unmarshal(e.Content, &updatePayload); err != nil { util.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{ "user_id": updatePayload.UserID, @@ -422,7 +423,7 @@ func (t *txnReq) processSigningKeyUpdate(ctx context.Context, e gomatrixserverli return nil } -// processReceiptEvent sends receipt events to the edu server +// processReceiptEvent sends receipt events to JetStream func (t *txnReq) processReceiptEvent(ctx context.Context, userID, roomID, receiptType string, timestamp gomatrixserverlib.Timestamp, @@ -430,17 +431,7 @@ func (t *txnReq) processReceiptEvent(ctx context.Context, ) error { // store every event for _, eventID := range eventIDs { - req := eduserverAPI.InputReceiptEventRequest{ - InputReceiptEvent: eduserverAPI.InputReceiptEvent{ - UserID: userID, - RoomID: roomID, - EventID: eventID, - Type: receiptType, - Timestamp: timestamp, - }, - } - resp := eduserverAPI.InputReceiptEventResponse{} - if err := t.eduAPI.InputReceiptEvent(ctx, &req, &resp); err != nil { + if err := t.producer.SendReceipt(ctx, userID, roomID, eventID, receiptType, timestamp); err != nil { return fmt.Errorf("unable to set receipt event: %w", err) } } diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index 4280643e9..8d2d85040 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -7,7 +7,6 @@ import ( "testing" "time" - eduAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/test" "github.com/matrix-org/dendrite/roomserver/api" @@ -53,44 +52,6 @@ func init() { } } -type testEDUProducer struct { - // this producer keeps track of calls to InputTypingEvent - invocations []eduAPI.InputTypingEventRequest -} - -func (p *testEDUProducer) InputTypingEvent( - ctx context.Context, - request *eduAPI.InputTypingEventRequest, - response *eduAPI.InputTypingEventResponse, -) error { - p.invocations = append(p.invocations, *request) - return nil -} - -func (p *testEDUProducer) InputSendToDeviceEvent( - ctx context.Context, - request *eduAPI.InputSendToDeviceEventRequest, - response *eduAPI.InputSendToDeviceEventResponse, -) error { - return nil -} - -func (o *testEDUProducer) InputReceiptEvent( - ctx context.Context, - request *eduAPI.InputReceiptEventRequest, - response *eduAPI.InputReceiptEventResponse, -) error { - return nil -} - -func (o *testEDUProducer) InputCrossSigningKeyUpdate( - ctx context.Context, - request *eduAPI.InputCrossSigningKeyUpdateRequest, - response *eduAPI.InputCrossSigningKeyUpdateResponse, -) error { - return nil -} - type testRoomserverAPI struct { api.RoomserverInternalAPITrace inputRoomEvents []api.InputRoomEvent @@ -225,7 +186,6 @@ func (c *txnFedClient) LookupMissingEvents(ctx context.Context, s gomatrixserver func mustCreateTransaction(rsAPI api.RoomserverInternalAPI, fedClient txnFederationClient, pdus []json.RawMessage) *txnReq { t := &txnReq{ rsAPI: rsAPI, - eduAPI: &testEDUProducer{}, keys: &test.NopJSONVerifier{}, federation: fedClient, roomsMu: internal.NewMutexByRoom(), diff --git a/federationapi/types/types.go b/federationapi/types/types.go index c486c05c4..a28a80b2f 100644 --- a/federationapi/types/types.go +++ b/federationapi/types/types.go @@ -18,6 +18,8 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) +const MSigningKeyUpdate = "m.signing_key_update" // TODO: move to gomatrixserverlib + // A JoinedHost is a server that is joined to a matrix room. type JoinedHost struct { // The MemberEventID of a m.room.member join event. @@ -51,3 +53,16 @@ type InboundPeek struct { RenewedTimestamp int64 RenewalInterval int64 } + +type FederationReceiptMRead struct { + User map[string]FederationReceiptData `json:"m.read"` +} + +type FederationReceiptData struct { + Data ReceiptTS `json:"data"` + EventIDs []string `json:"event_ids"` +} + +type ReceiptTS struct { + TS gomatrixserverlib.Timestamp `json:"ts"` +} diff --git a/eduserver/cache/cache.go b/internal/caching/cache_typing.go similarity index 97% rename from eduserver/cache/cache.go rename to internal/caching/cache_typing.go index f637d7c97..bd6a5fc1b 100644 --- a/eduserver/cache/cache.go +++ b/internal/caching/cache_typing.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cache +package caching import ( "sync" @@ -53,8 +53,8 @@ func (t *EDUCache) newRoomData() *roomData { } } -// New returns a new EDUCache initialised for use. -func New() *EDUCache { +// NewTypingCache returns a new EDUCache initialised for use. +func NewTypingCache() *EDUCache { return &EDUCache{data: make(map[string]*roomData)} } diff --git a/eduserver/cache/cache_test.go b/internal/caching/cache_typing_test.go similarity index 97% rename from eduserver/cache/cache_test.go rename to internal/caching/cache_typing_test.go index c7d01879f..c03d89bc3 100644 --- a/eduserver/cache/cache_test.go +++ b/internal/caching/cache_typing_test.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cache +package caching import ( "testing" @@ -24,9 +24,9 @@ import ( ) func TestEDUCache(t *testing.T) { - tCache := New() + tCache := NewTypingCache() if tCache == nil { - t.Fatal("New failed") + t.Fatal("NewTypingCache failed") } t.Run("AddTypingUser", func(t *testing.T) { diff --git a/internal/test/config.go b/internal/test/config.go index 0372fb9c6..0b0e897b8 100644 --- a/internal/test/config.go +++ b/internal/test/config.go @@ -97,7 +97,6 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(database) cfg.AppServiceAPI.InternalAPI.Listen = assignAddress() - cfg.EDUServer.InternalAPI.Listen = assignAddress() cfg.FederationAPI.InternalAPI.Listen = assignAddress() cfg.KeyServer.InternalAPI.Listen = assignAddress() cfg.MediaAPI.InternalAPI.Listen = assignAddress() @@ -106,7 +105,6 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con cfg.UserAPI.InternalAPI.Listen = assignAddress() cfg.AppServiceAPI.InternalAPI.Connect = cfg.AppServiceAPI.InternalAPI.Listen - cfg.EDUServer.InternalAPI.Connect = cfg.EDUServer.InternalAPI.Listen cfg.FederationAPI.InternalAPI.Connect = cfg.FederationAPI.InternalAPI.Listen cfg.KeyServer.InternalAPI.Connect = cfg.KeyServer.InternalAPI.Listen cfg.MediaAPI.InternalAPI.Connect = cfg.MediaAPI.InternalAPI.Listen diff --git a/keyserver/api/api.go b/keyserver/api/api.go index d361c6222..429617b10 100644 --- a/keyserver/api/api.go +++ b/keyserver/api/api.go @@ -21,7 +21,6 @@ import ( "strings" "time" - eduapi "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/keyserver/types" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" @@ -66,14 +65,25 @@ const ( // DeviceMessage represents the message produced into Kafka by the key server. type DeviceMessage struct { - Type DeviceMessageType `json:"Type,omitempty"` - *DeviceKeys `json:"DeviceKeys,omitempty"` - *eduapi.OutputCrossSigningKeyUpdate `json:"CrossSigningKeyUpdate,omitempty"` + Type DeviceMessageType `json:"Type,omitempty"` + *DeviceKeys `json:"DeviceKeys,omitempty"` + *OutputCrossSigningKeyUpdate `json:"CrossSigningKeyUpdate,omitempty"` // A monotonically increasing number which represents device changes for this user. StreamID int64 DeviceChangeID int64 } +// OutputCrossSigningKeyUpdate is an entry in the signing key update output kafka log +type OutputCrossSigningKeyUpdate struct { + CrossSigningKeyUpdate `json:"signing_keys"` +} + +type CrossSigningKeyUpdate struct { + MasterKey *gomatrixserverlib.CrossSigningKey `json:"master_key,omitempty"` + SelfSigningKey *gomatrixserverlib.CrossSigningKey `json:"self_signing_key,omitempty"` + UserID string `json:"user_id"` +} + // DeviceKeysEqual returns true if the device keys updates contain the // same display name and key JSON. This will return false if either of // the updates is not a device keys update, or if the user ID/device ID diff --git a/keyserver/internal/cross_signing.go b/keyserver/internal/cross_signing.go index 5124f37e6..0d083b4ba 100644 --- a/keyserver/internal/cross_signing.go +++ b/keyserver/internal/cross_signing.go @@ -22,7 +22,6 @@ import ( "fmt" "strings" - eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/keyserver/types" "github.com/matrix-org/gomatrixserverlib" @@ -246,7 +245,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P } // Finally, generate a notification that we updated the keys. - update := eduserverAPI.CrossSigningKeyUpdate{ + update := api.CrossSigningKeyUpdate{ UserID: req.UserID, } if mk, ok := byPurpose[gomatrixserverlib.CrossSigningKeyPurposeMaster]; ok { @@ -337,7 +336,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceSignatures(ctx context.Context, req for userID := range req.Signatures { masterKey := queryRes.MasterKeys[userID] selfSigningKey := queryRes.SelfSigningKeys[userID] - update := eduserverAPI.CrossSigningKeyUpdate{ + update := api.CrossSigningKeyUpdate{ UserID: userID, MasterKey: &masterKey, SelfSigningKey: &selfSigningKey, diff --git a/keyserver/producers/keychange.go b/keyserver/producers/keychange.go index 9e1c4c645..f86c34177 100644 --- a/keyserver/producers/keychange.go +++ b/keyserver/producers/keychange.go @@ -18,7 +18,6 @@ import ( "context" "encoding/json" - eduapi "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/keyserver/storage" "github.com/matrix-org/dendrite/setup/jetstream" @@ -70,10 +69,10 @@ func (p *KeyChange) ProduceKeyChanges(keys []api.DeviceMessage) error { return nil } -func (p *KeyChange) ProduceSigningKeyUpdate(key eduapi.CrossSigningKeyUpdate) error { +func (p *KeyChange) ProduceSigningKeyUpdate(key api.CrossSigningKeyUpdate) error { output := &api.DeviceMessage{ Type: api.TypeCrossSigningUpdate, - OutputCrossSigningKeyUpdate: &eduapi.OutputCrossSigningKeyUpdate{ + OutputCrossSigningKeyUpdate: &api.OutputCrossSigningKeyUpdate{ CrossSigningKeyUpdate: key, }, } diff --git a/setup/base/base.go b/setup/base/base.go index 6135e080e..43d613b0c 100644 --- a/setup/base/base.go +++ b/setup/base/base.go @@ -45,8 +45,6 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" asinthttp "github.com/matrix-org/dendrite/appservice/inthttp" - eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" - eduinthttp "github.com/matrix-org/dendrite/eduserver/inthttp" federationAPI "github.com/matrix-org/dendrite/federationapi/api" federationIntHTTP "github.com/matrix-org/dendrite/federationapi/inthttp" keyserverAPI "github.com/matrix-org/dendrite/keyserver/api" @@ -247,15 +245,6 @@ func (b *BaseDendrite) UserAPIClient() userapi.UserInternalAPI { return userAPI } -// EDUServerClient returns EDUServerInputAPI for hitting the EDU server over HTTP -func (b *BaseDendrite) EDUServerClient() eduServerAPI.EDUServerInputAPI { - e, err := eduinthttp.NewEDUServerClient(b.Cfg.EDUServerURL(), b.apiHttpClient) - if err != nil { - logrus.WithError(err).Panic("EDUServerClient failed", b.apiHttpClient) - } - return e -} - // FederationAPIHTTPClient returns FederationInternalAPI for hitting // the federation API server over HTTP func (b *BaseDendrite) FederationAPIHTTPClient() federationAPI.FederationInternalAPI { diff --git a/setup/config/config.go b/setup/config/config.go index eb371a54b..e03518e24 100644 --- a/setup/config/config.go +++ b/setup/config/config.go @@ -56,7 +56,6 @@ type Dendrite struct { Global Global `yaml:"global"` AppServiceAPI AppServiceAPI `yaml:"app_service_api"` ClientAPI ClientAPI `yaml:"client_api"` - EDUServer EDUServer `yaml:"edu_server"` FederationAPI FederationAPI `yaml:"federation_api"` KeyServer KeyServer `yaml:"key_server"` MediaAPI MediaAPI `yaml:"media_api"` @@ -296,7 +295,6 @@ func (c *Dendrite) Defaults(generate bool) { c.Global.Defaults(generate) c.ClientAPI.Defaults(generate) - c.EDUServer.Defaults(generate) c.FederationAPI.Defaults(generate) c.KeyServer.Defaults(generate) c.MediaAPI.Defaults(generate) @@ -314,8 +312,7 @@ func (c *Dendrite) Verify(configErrs *ConfigErrors, isMonolith bool) { Verify(configErrs *ConfigErrors, isMonolith bool) } for _, c := range []verifiable{ - &c.Global, &c.ClientAPI, - &c.EDUServer, &c.FederationAPI, + &c.Global, &c.ClientAPI, &c.FederationAPI, &c.KeyServer, &c.MediaAPI, &c.RoomServer, &c.SyncAPI, &c.UserAPI, &c.AppServiceAPI, &c.MSCs, @@ -327,7 +324,6 @@ func (c *Dendrite) Verify(configErrs *ConfigErrors, isMonolith bool) { func (c *Dendrite) Wiring() { c.Global.JetStream.Matrix = &c.Global c.ClientAPI.Matrix = &c.Global - c.EDUServer.Matrix = &c.Global c.FederationAPI.Matrix = &c.Global c.KeyServer.Matrix = &c.Global c.MediaAPI.Matrix = &c.Global @@ -519,15 +515,6 @@ func (config *Dendrite) UserAPIURL() string { return string(config.UserAPI.InternalAPI.Connect) } -// EDUServerURL returns an HTTP URL for where the EDU server is listening. -func (config *Dendrite) EDUServerURL() string { - // Hard code the EDU server to talk HTTP for now. - // If we support HTTPS we need to think of a practical way to do certificate validation. - // People setting up servers shouldn't need to get a certificate valid for the public - // internet for an internal API. - return string(config.EDUServer.InternalAPI.Connect) -} - // KeyServerURL returns an HTTP URL for where the key server is listening. func (config *Dendrite) KeyServerURL() string { // Hard code the key server to talk HTTP for now. diff --git a/setup/config/config_eduserver.go b/setup/config/config_eduserver.go deleted file mode 100644 index e7ed36aa0..000000000 --- a/setup/config/config_eduserver.go +++ /dev/null @@ -1,17 +0,0 @@ -package config - -type EDUServer struct { - Matrix *Global `yaml:"-"` - - InternalAPI InternalAPIOptions `yaml:"internal_api"` -} - -func (c *EDUServer) Defaults(generate bool) { - c.InternalAPI.Listen = "http://localhost:7778" - c.InternalAPI.Connect = "http://localhost:7778" -} - -func (c *EDUServer) Verify(configErrs *ConfigErrors, isMonolith bool) { - checkURL(configErrs, "edu_server.internal_api.listen", string(c.InternalAPI.Listen)) - checkURL(configErrs, "edu_server.internal_api.connect", string(c.InternalAPI.Connect)) -} diff --git a/setup/config/config_test.go b/setup/config/config_test.go index e6f0a493e..46e973fac 100644 --- a/setup/config/config_test.go +++ b/setup/config/config_test.go @@ -101,10 +101,6 @@ current_state_server: max_open_conns: 100 max_idle_conns: 2 conn_max_lifetime: -1 -edu_server: - internal_api: - listen: http://localhost:7778 - connect: http://localhost:7778 federation_api: internal_api: listen: http://localhost:7772 diff --git a/setup/jetstream/nats.go b/setup/jetstream/nats.go index 328cf9155..4e4fe7a29 100644 --- a/setup/jetstream/nats.go +++ b/setup/jetstream/nats.go @@ -157,5 +157,26 @@ func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsc } } + // Clean up old consumers so that interest-based consumers do the + // right thing. + for stream, consumers := range map[string][]string{ + OutputClientData: {"SyncAPIClientAPIConsumer"}, + OutputReceiptEvent: {"SyncAPIEDUServerReceiptConsumer", "FederationAPIEDUServerConsumer"}, + OutputSendToDeviceEvent: {"SyncAPIEDUServerSendToDeviceConsumer", "FederationAPIEDUServerConsumer"}, + OutputTypingEvent: {"SyncAPIEDUServerTypingConsumer", "FederationAPIEDUServerConsumer"}, + } { + streamName := cfg.Matrix.JetStream.Prefixed(stream) + for _, consumer := range consumers { + consumerName := cfg.Matrix.JetStream.Prefixed(consumer) + "Pull" + consumerInfo, err := s.ConsumerInfo(streamName, consumerName) + if err != nil || consumerInfo == nil { + continue + } + if err = s.DeleteConsumer(streamName, consumerName); err != nil { + logrus.WithError(err).Errorf("Unable to clean up old consumer %q for stream %q", consumer, stream) + } + } + } + return s, nc } diff --git a/setup/jetstream/streams.go b/setup/jetstream/streams.go index aa979924b..5f0d37fdc 100644 --- a/setup/jetstream/streams.go +++ b/setup/jetstream/streams.go @@ -9,8 +9,9 @@ import ( ) const ( - UserID = "user_id" - RoomID = "room_id" + UserID = "user_id" + RoomID = "room_id" + EventID = "event_id" ) var ( diff --git a/setup/monolith.go b/setup/monolith.go index cf6872f9c..32f1a6494 100644 --- a/setup/monolith.go +++ b/setup/monolith.go @@ -19,7 +19,6 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi" "github.com/matrix-org/dendrite/clientapi/api" - eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/federationapi" federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/transactions" @@ -43,12 +42,11 @@ type Monolith struct { Client *gomatrixserverlib.Client FedClient *gomatrixserverlib.FederationClient - AppserviceAPI appserviceAPI.AppServiceQueryAPI - EDUInternalAPI eduServerAPI.EDUServerInputAPI - FederationAPI federationAPI.FederationInternalAPI - RoomserverAPI roomserverAPI.RoomserverInternalAPI - UserAPI userapi.UserInternalAPI - KeyAPI keyAPI.KeyInternalAPI + AppserviceAPI appserviceAPI.AppServiceQueryAPI + FederationAPI federationAPI.FederationInternalAPI + RoomserverAPI roomserverAPI.RoomserverInternalAPI + UserAPI userapi.UserInternalAPI + KeyAPI keyAPI.KeyInternalAPI // Optional ExtPublicRoomsProvider api.ExtraPublicRoomsProvider @@ -64,14 +62,14 @@ func (m *Monolith) AddAllPublicRoutes(process *process.ProcessContext, csMux, ss clientapi.AddPublicRoutes( process, csMux, synapseMux, &m.Config.ClientAPI, m.FedClient, m.RoomserverAPI, - m.EDUInternalAPI, m.AppserviceAPI, transactions.New(), + m.AppserviceAPI, transactions.New(), m.FederationAPI, m.UserAPI, userDirectoryProvider, m.KeyAPI, m.ExtPublicRoomsProvider, &m.Config.MSCs, ) federationapi.AddPublicRoutes( - ssMux, keyMux, wkMux, &m.Config.FederationAPI, m.UserAPI, m.FedClient, + process, ssMux, keyMux, wkMux, &m.Config.FederationAPI, m.UserAPI, m.FedClient, m.KeyRing, m.RoomserverAPI, m.FederationAPI, - m.EDUInternalAPI, m.KeyAPI, &m.Config.MSCs, nil, + m.KeyAPI, &m.Config.MSCs, nil, ) mediaapi.AddPublicRoutes(mediaMux, &m.Config.MediaAPI, &m.Config.ClientAPI.RateLimiting, m.UserAPI, m.Client) syncapi.AddPublicRoutes( diff --git a/syncapi/consumers/clientapi.go b/syncapi/consumers/clientapi.go index 40c1cd3d6..c28da4600 100644 --- a/syncapi/consumers/clientapi.go +++ b/syncapi/consumers/clientapi.go @@ -62,7 +62,7 @@ func NewOutputClientDataConsumer( ctx: process.Context(), jetstream: js, topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputClientData), - durable: cfg.Matrix.JetStream.Durable("SyncAPIClientAPIConsumer"), + durable: cfg.Matrix.JetStream.Durable("SyncAPIAccountDataConsumer"), db: store, notifier: notifier, stream: stream, diff --git a/syncapi/consumers/eduserver_receipts.go b/syncapi/consumers/receipts.go similarity index 87% rename from syncapi/consumers/eduserver_receipts.go rename to syncapi/consumers/receipts.go index ab79998ea..6bb0747f0 100644 --- a/syncapi/consumers/eduserver_receipts.go +++ b/syncapi/consumers/receipts.go @@ -17,11 +17,10 @@ package consumers import ( "context" "database/sql" - "encoding/json" "fmt" + "strconv" "github.com/getsentry/sentry-go" - "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" @@ -63,7 +62,7 @@ func NewOutputReceiptEventConsumer( ctx: process.Context(), jetstream: js, topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent), - durable: cfg.Matrix.JetStream.Durable("SyncAPIEDUServerReceiptConsumer"), + durable: cfg.Matrix.JetStream.Durable("SyncAPIReceiptConsumer"), db: store, notifier: notifier, stream: stream, @@ -72,7 +71,7 @@ func NewOutputReceiptEventConsumer( } } -// Start consuming from EDU api +// Start consuming receipts events. func (s *OutputReceiptEventConsumer) Start() error { return jetstream.JetStreamConsumer( s.ctx, s.jetstream, s.topic, s.durable, s.onMessage, @@ -81,14 +80,23 @@ func (s *OutputReceiptEventConsumer) Start() error { } func (s *OutputReceiptEventConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool { - var output api.OutputReceiptEvent - if err := json.Unmarshal(msg.Data, &output); err != nil { + output := types.OutputReceiptEvent{ + UserID: msg.Header.Get(jetstream.UserID), + RoomID: msg.Header.Get(jetstream.RoomID), + EventID: msg.Header.Get(jetstream.EventID), + Type: msg.Header.Get("type"), + } + + timestamp, err := strconv.Atoi(msg.Header.Get("timestamp")) + if err != nil { // If the message was invalid, log it and move on to the next message in the stream - log.WithError(err).Errorf("EDU server output log: message parse failure") + log.WithError(err).Errorf("output log: message parse failure") sentry.CaptureException(err) return true } + output.Timestamp = gomatrixserverlib.Timestamp(timestamp) + streamPos, err := s.db.StoreReceipt( s.ctx, output.RoomID, @@ -117,7 +125,7 @@ func (s *OutputReceiptEventConsumer) onMessage(ctx context.Context, msg *nats.Ms return true } -func (s *OutputReceiptEventConsumer) sendReadUpdate(ctx context.Context, output api.OutputReceiptEvent) error { +func (s *OutputReceiptEventConsumer) sendReadUpdate(ctx context.Context, output types.OutputReceiptEvent) error { if output.Type != "m.read" { return nil } diff --git a/syncapi/consumers/eduserver_sendtodevice.go b/syncapi/consumers/sendtodevice.go similarity index 87% rename from syncapi/consumers/eduserver_sendtodevice.go rename to syncapi/consumers/sendtodevice.go index bdbe77352..0b9153fcd 100644 --- a/syncapi/consumers/eduserver_sendtodevice.go +++ b/syncapi/consumers/sendtodevice.go @@ -19,7 +19,6 @@ import ( "encoding/json" "github.com/getsentry/sentry-go" - "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" @@ -58,7 +57,7 @@ func NewOutputSendToDeviceEventConsumer( ctx: process.Context(), jetstream: js, topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), - durable: cfg.Matrix.JetStream.Durable("SyncAPIEDUServerSendToDeviceConsumer"), + durable: cfg.Matrix.JetStream.Durable("SyncAPISendToDeviceConsumer"), db: store, serverName: cfg.Matrix.ServerName, notifier: notifier, @@ -66,7 +65,7 @@ func NewOutputSendToDeviceEventConsumer( } } -// Start consuming from EDU api +// Start consuming send-to-device events. func (s *OutputSendToDeviceEventConsumer) Start() error { return jetstream.JetStreamConsumer( s.ctx, s.jetstream, s.topic, s.durable, s.onMessage, @@ -75,15 +74,8 @@ func (s *OutputSendToDeviceEventConsumer) Start() error { } func (s *OutputSendToDeviceEventConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool { - var output api.OutputSendToDeviceEvent - if err := json.Unmarshal(msg.Data, &output); err != nil { - // If the message was invalid, log it and move on to the next message in the stream - log.WithError(err).Errorf("EDU server output log: message parse failure") - sentry.CaptureException(err) - return true - } - - _, domain, err := gomatrixserverlib.SplitID('@', output.UserID) + userID := msg.Header.Get(jetstream.UserID) + _, domain, err := gomatrixserverlib.SplitID('@', userID) if err != nil { sentry.CaptureException(err) return true @@ -92,12 +84,20 @@ func (s *OutputSendToDeviceEventConsumer) onMessage(ctx context.Context, msg *na return true } + var output types.OutputSendToDeviceEvent + if err = json.Unmarshal(msg.Data, &output); err != nil { + // If the message was invalid, log it and move on to the next message in the stream + log.WithError(err).Errorf("output log: message parse failure") + sentry.CaptureException(err) + return true + } + util.GetLogger(context.TODO()).WithFields(log.Fields{ "sender": output.Sender, "user_id": output.UserID, "device_id": output.DeviceID, "event_type": output.Type, - }).Info("sync API received send-to-device event from EDU server") + }).Debugf("sync API received send-to-device event from the clientapi/federationsender") streamPos, err := s.db.StoreNewSendForDeviceMessage( s.ctx, output.UserID, output.DeviceID, output.SendToDeviceEvent, diff --git a/syncapi/consumers/eduserver_typing.go b/syncapi/consumers/typing.go similarity index 67% rename from syncapi/consumers/eduserver_typing.go rename to syncapi/consumers/typing.go index c2828c7fc..48e484ec5 100644 --- a/syncapi/consumers/eduserver_typing.go +++ b/syncapi/consumers/typing.go @@ -16,16 +16,14 @@ package consumers import ( "context" - "encoding/json" + "strconv" + "time" - "github.com/getsentry/sentry-go" - "github.com/matrix-org/dendrite/eduserver/api" - "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/syncapi/notifier" - "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" "github.com/nats-io/nats.go" log "github.com/sirupsen/logrus" @@ -37,7 +35,7 @@ type OutputTypingEventConsumer struct { jetstream nats.JetStreamContext durable string topic string - eduCache *cache.EDUCache + eduCache *caching.EDUCache stream types.StreamProvider notifier *notifier.Notifier } @@ -48,8 +46,7 @@ func NewOutputTypingEventConsumer( process *process.ProcessContext, cfg *config.SyncAPI, js nats.JetStreamContext, - store storage.Database, - eduCache *cache.EDUCache, + eduCache *caching.EDUCache, notifier *notifier.Notifier, stream types.StreamProvider, ) *OutputTypingEventConsumer { @@ -57,14 +54,14 @@ func NewOutputTypingEventConsumer( ctx: process.Context(), jetstream: js, topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent), - durable: cfg.Matrix.JetStream.Durable("SyncAPIEDUServerTypingConsumer"), + durable: cfg.Matrix.JetStream.Durable("SyncAPITypingConsumer"), eduCache: eduCache, notifier: notifier, stream: stream, } } -// Start consuming from EDU api +// Start consuming typing events. func (s *OutputTypingEventConsumer) Start() error { return jetstream.JetStreamConsumer( s.ctx, s.jetstream, s.topic, s.durable, s.onMessage, @@ -73,34 +70,40 @@ func (s *OutputTypingEventConsumer) Start() error { } func (s *OutputTypingEventConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool { - var output api.OutputTypingEvent - if err := json.Unmarshal(msg.Data, &output); err != nil { - // If the message was invalid, log it and move on to the next message in the stream - log.WithError(err).Errorf("EDU server output log: message parse failure") - sentry.CaptureException(err) + roomID := msg.Header.Get(jetstream.RoomID) + userID := msg.Header.Get(jetstream.UserID) + typing, err := strconv.ParseBool(msg.Header.Get("typing")) + if err != nil { + log.WithError(err).Errorf("output log: typing parse failure") + return true + } + timeout, err := strconv.Atoi(msg.Header.Get("timeout_ms")) + if err != nil { + log.WithError(err).Errorf("output log: timeout_ms parse failure") return true } log.WithFields(log.Fields{ - "room_id": output.Event.RoomID, - "user_id": output.Event.UserID, - "typing": output.Event.Typing, - }).Debug("received data from EDU server") + "room_id": roomID, + "user_id": userID, + "typing": typing, + "timeout": timeout, + }).Debug("syncapi received EDU data from client api") var typingPos types.StreamPosition - typingEvent := output.Event - if typingEvent.Typing { + if typing { + expiry := time.Now().Add(time.Duration(timeout) * time.Millisecond) typingPos = types.StreamPosition( - s.eduCache.AddTypingUser(typingEvent.UserID, typingEvent.RoomID, output.ExpireTime), + s.eduCache.AddTypingUser(userID, roomID, &expiry), ) } else { typingPos = types.StreamPosition( - s.eduCache.RemoveUser(typingEvent.UserID, typingEvent.RoomID), + s.eduCache.RemoveUser(userID, roomID), ) } s.stream.Advance(typingPos) - s.notifier.OnNewTyping(output.Event.RoomID, types.StreamingToken{TypingPosition: typingPos}) + s.notifier.OnNewTyping(roomID, types.StreamingToken{TypingPosition: typingPos}) return true } diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index b6ac5be19..647fffad5 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -17,7 +17,6 @@ package storage import ( "context" - eduAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" @@ -46,7 +45,7 @@ type Database interface { InviteEventsInRange(ctx context.Context, targetUserID string, r types.Range) (map[string]*gomatrixserverlib.HeaderedEvent, map[string]*gomatrixserverlib.HeaderedEvent, error) PeeksInRange(ctx context.Context, userID, deviceID string, r types.Range) (peeks []types.Peek, err error) - RoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []eduAPI.OutputReceiptEvent, error) + RoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []types.OutputReceiptEvent, error) // AllJoinedUsersInRooms returns a map of room ID to a list of all joined user IDs. AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) @@ -136,7 +135,7 @@ type Database interface { // StoreReceipt stores new receipt events StoreReceipt(ctx context.Context, roomId, receiptType, userId, eventId string, timestamp gomatrixserverlib.Timestamp) (pos types.StreamPosition, err error) // GetRoomReceipts gets all receipts for a given roomID - GetRoomReceipts(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) ([]eduAPI.OutputReceiptEvent, error) + GetRoomReceipts(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) ([]types.OutputReceiptEvent, error) // UpsertRoomUnreadNotificationCounts updates the notification statistics about a (user, room) key. UpsertRoomUnreadNotificationCounts(ctx context.Context, userID, roomID string, notificationCount, highlightCount int) (types.StreamPosition, error) diff --git a/syncapi/storage/postgres/receipt_table.go b/syncapi/storage/postgres/receipt_table.go index 474d0c020..2a42ffd74 100644 --- a/syncapi/storage/postgres/receipt_table.go +++ b/syncapi/storage/postgres/receipt_table.go @@ -21,7 +21,6 @@ import ( "github.com/lib/pq" - "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" @@ -95,16 +94,16 @@ func (r *receiptStatements) UpsertReceipt(ctx context.Context, txn *sql.Tx, room return } -func (r *receiptStatements) SelectRoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []api.OutputReceiptEvent, error) { +func (r *receiptStatements) SelectRoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []types.OutputReceiptEvent, error) { var lastPos types.StreamPosition rows, err := r.selectRoomReceipts.QueryContext(ctx, pq.Array(roomIDs), streamPos) if err != nil { return 0, nil, fmt.Errorf("unable to query room receipts: %w", err) } defer internal.CloseAndLogIfError(ctx, rows, "SelectRoomReceiptsAfter: rows.close() failed") - var res []api.OutputReceiptEvent + var res []types.OutputReceiptEvent for rows.Next() { - r := api.OutputReceiptEvent{} + r := types.OutputReceiptEvent{} var id types.StreamPosition err = rows.Scan(&id, &r.RoomID, &r.Type, &r.UserID, &r.EventID, &r.Timestamp) if err != nil { diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index 9a2dc0d44..349e44526 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -20,7 +20,6 @@ import ( "encoding/json" "fmt" - eduAPI "github.com/matrix-org/dendrite/eduserver/api" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/internal/eventutil" @@ -135,7 +134,7 @@ func (d *Database) PeeksInRange(ctx context.Context, userID, deviceID string, r return d.Peeks.SelectPeeksInRange(ctx, nil, userID, deviceID, r) } -func (d *Database) RoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []eduAPI.OutputReceiptEvent, error) { +func (d *Database) RoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []types.OutputReceiptEvent, error) { return d.Receipts.SelectRoomReceiptsAfter(ctx, roomIDs, streamPos) } @@ -972,7 +971,7 @@ func (d *Database) StoreReceipt(ctx context.Context, roomId, receiptType, userId return } -func (d *Database) GetRoomReceipts(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) ([]eduAPI.OutputReceiptEvent, error) { +func (d *Database) GetRoomReceipts(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) ([]types.OutputReceiptEvent, error) { _, receipts, err := d.Receipts.SelectRoomReceiptsAfter(ctx, roomIDs, streamPos) return receipts, err } diff --git a/syncapi/storage/sqlite3/receipt_table.go b/syncapi/storage/sqlite3/receipt_table.go index 9111a39f6..dea057719 100644 --- a/syncapi/storage/sqlite3/receipt_table.go +++ b/syncapi/storage/sqlite3/receipt_table.go @@ -20,7 +20,6 @@ import ( "fmt" "strings" - "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" @@ -99,7 +98,7 @@ func (r *receiptStatements) UpsertReceipt(ctx context.Context, txn *sql.Tx, room } // SelectRoomReceiptsAfter select all receipts for a given room after a specific timestamp -func (r *receiptStatements) SelectRoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []api.OutputReceiptEvent, error) { +func (r *receiptStatements) SelectRoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []types.OutputReceiptEvent, error) { selectSQL := strings.Replace(selectRoomReceipts, "($2)", sqlutil.QueryVariadicOffset(len(roomIDs), 1), 1) var lastPos types.StreamPosition params := make([]interface{}, len(roomIDs)+1) @@ -112,9 +111,9 @@ func (r *receiptStatements) SelectRoomReceiptsAfter(ctx context.Context, roomIDs return 0, nil, fmt.Errorf("unable to query room receipts: %w", err) } defer internal.CloseAndLogIfError(ctx, rows, "SelectRoomReceiptsAfter: rows.close() failed") - var res []api.OutputReceiptEvent + var res []types.OutputReceiptEvent for rows.Next() { - r := api.OutputReceiptEvent{} + r := types.OutputReceiptEvent{} var id types.StreamPosition err = rows.Scan(&id, &r.RoomID, &r.Type, &r.UserID, &r.EventID, &r.Timestamp) if err != nil { diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index 640b7dc31..ba0076e22 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -18,7 +18,6 @@ import ( "context" "database/sql" - eduAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/types" @@ -168,7 +167,7 @@ type Filter interface { type Receipts interface { UpsertReceipt(ctx context.Context, txn *sql.Tx, roomId, receiptType, userId, eventId string, timestamp gomatrixserverlib.Timestamp) (pos types.StreamPosition, err error) - SelectRoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []eduAPI.OutputReceiptEvent, error) + SelectRoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []types.OutputReceiptEvent, error) SelectMaxReceiptID(ctx context.Context, txn *sql.Tx) (id int64, err error) } diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go index 35ffd3a1e..680f8cd8e 100644 --- a/syncapi/streams/stream_receipt.go +++ b/syncapi/streams/stream_receipt.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" - eduAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -53,7 +52,7 @@ func (p *ReceiptStreamProvider) IncrementalSync( } // Group receipts by room, so we can create one ClientEvent for every room - receiptsByRoom := make(map[string][]eduAPI.OutputReceiptEvent) + receiptsByRoom := make(map[string][]types.OutputReceiptEvent) for _, receipt := range receipts { receiptsByRoom[receipt.RoomID] = append(receiptsByRoom[receipt.RoomID], receipt) } @@ -68,15 +67,15 @@ func (p *ReceiptStreamProvider) IncrementalSync( Type: gomatrixserverlib.MReceipt, RoomID: roomID, } - content := make(map[string]eduAPI.ReceiptMRead) + content := make(map[string]ReceiptMRead) for _, receipt := range receipts { read, ok := content[receipt.EventID] if !ok { - read = eduAPI.ReceiptMRead{ - User: make(map[string]eduAPI.ReceiptTS), + read = ReceiptMRead{ + User: make(map[string]ReceiptTS), } } - read.User[receipt.UserID] = eduAPI.ReceiptTS{TS: receipt.Timestamp} + read.User[receipt.UserID] = ReceiptTS{TS: receipt.Timestamp} content[receipt.EventID] = read } ev.Content, err = json.Marshal(content) @@ -91,3 +90,11 @@ func (p *ReceiptStreamProvider) IncrementalSync( return lastPos } + +type ReceiptMRead struct { + User map[string]ReceiptTS `json:"m.read"` +} + +type ReceiptTS struct { + TS gomatrixserverlib.Timestamp `json:"ts"` +} diff --git a/syncapi/streams/stream_typing.go b/syncapi/streams/stream_typing.go index 1e7a46bdc..e46cd447b 100644 --- a/syncapi/streams/stream_typing.go +++ b/syncapi/streams/stream_typing.go @@ -4,14 +4,14 @@ import ( "context" "encoding/json" - "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) type TypingStreamProvider struct { StreamProvider - EDUCache *cache.EDUCache + EDUCache *caching.EDUCache } func (p *TypingStreamProvider) CompleteSync( diff --git a/syncapi/streams/streams.go b/syncapi/streams/streams.go index 17951acb4..b2273aadb 100644 --- a/syncapi/streams/streams.go +++ b/syncapi/streams/streams.go @@ -3,7 +3,7 @@ package streams import ( "context" - "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/internal/caching" keyapi "github.com/matrix-org/dendrite/keyserver/api" rsapi "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage" @@ -25,7 +25,7 @@ type Streams struct { func NewSyncStreamProviders( d storage.Database, userAPI userapi.UserInternalAPI, rsAPI rsapi.RoomserverInternalAPI, keyAPI keyapi.KeyInternalAPI, - eduCache *cache.EDUCache, + eduCache *caching.EDUCache, ) *Streams { streams := &Streams{ PDUStreamProvider: &PDUStreamProvider{ diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index ed8118bfc..b579467ae 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -18,9 +18,9 @@ import ( "context" "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/internal/caching" "github.com/sirupsen/logrus" - "github.com/matrix-org/dendrite/eduserver/cache" keyapi "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" @@ -56,7 +56,7 @@ func AddPublicRoutes( logrus.WithError(err).Panicf("failed to connect to sync db") } - eduCache := cache.New() + eduCache := caching.NewTypingCache() streams := streams.NewSyncStreamProviders(syncDB, userAPI, rsAPI, keyAPI, eduCache) notifier := notifier.NewNotifier(streams.Latest(context.Background())) if err = notifier.Load(context.Background(), syncDB); err != nil { @@ -110,7 +110,7 @@ func AddPublicRoutes( } typingConsumer := consumers.NewOutputTypingEventConsumer( - process, cfg, js, syncDB, eduCache, notifier, streams.TypingStreamProvider, + process, cfg, js, eduCache, notifier, streams.TypingStreamProvider, ) if err = typingConsumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start typing consumer") diff --git a/syncapi/types/types.go b/syncapi/types/types.go index 4150e6c98..f964b80b5 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -487,3 +487,21 @@ type StreamedEvent struct { Event *gomatrixserverlib.HeaderedEvent `json:"event"` StreamPosition StreamPosition `json:"stream_position"` } + +// OutputReceiptEvent is an entry in the receipt output kafka log +type OutputReceiptEvent struct { + UserID string `json:"user_id"` + RoomID string `json:"room_id"` + EventID string `json:"event_id"` + Type string `json:"type"` + Timestamp gomatrixserverlib.Timestamp `json:"timestamp"` +} + +// OutputSendToDeviceEvent is an entry in the send-to-device output kafka log. +// This contains the full event content, along with the user ID and device ID +// to which it is destined. +type OutputSendToDeviceEvent struct { + UserID string `json:"user_id"` + DeviceID string `json:"device_id"` + gomatrixserverlib.SendToDeviceEvent +} From 8213b2ba3071c2426cfcc506bafc6e45e23c9925 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 30 Mar 2022 15:01:22 +0100 Subject: [PATCH 39/55] Update Pinecone P2P demo --- build/gobind-pinecone/monolith.go | 15 ++++++++++ build/gobind-pinecone/platform_ios.go | 14 ++++++++++ build/gobind-pinecone/platform_other.go | 14 ++++++++++ cmd/dendrite-demo-pinecone/conn/client.go | 14 ++++++++++ cmd/dendrite-demo-pinecone/conn/ws.go | 14 ++++++++++ .../defaults/defaults.go | 21 ++++++++++++++ .../embed/embed_elementweb.go | 14 ++++++++++ .../embed/embed_other.go | 14 ++++++++++ cmd/dendrite-demo-pinecone/main.go | 3 +- cmd/dendrite-demo-pinecone/rooms/rooms.go | 14 ++++++---- cmd/dendrite-demo-pinecone/users/users.go | 28 +++++++++++++++---- go.mod | 2 +- go.sum | 4 +-- 13 files changed, 157 insertions(+), 14 deletions(-) create mode 100644 cmd/dendrite-demo-pinecone/defaults/defaults.go diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index efc21f596..97ce08d82 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -1,3 +1,17 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package gobind import ( @@ -353,6 +367,7 @@ func (m *DendriteMonolith) Start() { httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux) httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux) httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux) + httpRouter.HandleFunc("/pinecone", m.PineconeRouter.ManholeHandler) pMux := mux.NewRouter().SkipClean(true).UseEncodedPath() pMux.PathPrefix(users.PublicURL).HandlerFunc(userProvider.FederatedUserProfiles) diff --git a/build/gobind-pinecone/platform_ios.go b/build/gobind-pinecone/platform_ios.go index 802d7faca..a89ebfcd0 100644 --- a/build/gobind-pinecone/platform_ios.go +++ b/build/gobind-pinecone/platform_ios.go @@ -1,3 +1,17 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build ios // +build ios diff --git a/build/gobind-pinecone/platform_other.go b/build/gobind-pinecone/platform_other.go index 2e81e2f43..2793026b8 100644 --- a/build/gobind-pinecone/platform_other.go +++ b/build/gobind-pinecone/platform_other.go @@ -1,3 +1,17 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build !ios // +build !ios diff --git a/cmd/dendrite-demo-pinecone/conn/client.go b/cmd/dendrite-demo-pinecone/conn/client.go index e3cc0468c..59de9690c 100644 --- a/cmd/dendrite-demo-pinecone/conn/client.go +++ b/cmd/dendrite-demo-pinecone/conn/client.go @@ -1,3 +1,17 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package conn import ( diff --git a/cmd/dendrite-demo-pinecone/conn/ws.go b/cmd/dendrite-demo-pinecone/conn/ws.go index ef403e290..ed85abd51 100644 --- a/cmd/dendrite-demo-pinecone/conn/ws.go +++ b/cmd/dendrite-demo-pinecone/conn/ws.go @@ -1,3 +1,17 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package conn import ( diff --git a/cmd/dendrite-demo-pinecone/defaults/defaults.go b/cmd/dendrite-demo-pinecone/defaults/defaults.go new file mode 100644 index 000000000..c92493137 --- /dev/null +++ b/cmd/dendrite-demo-pinecone/defaults/defaults.go @@ -0,0 +1,21 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package defaults + +import "github.com/matrix-org/gomatrixserverlib" + +var DefaultServerNames = map[gomatrixserverlib.ServerName]struct{}{ + "3bf0258d23c60952639cc4c69c71d1508a7d43a0475d9000ff900a1848411ec7": {}, +} diff --git a/cmd/dendrite-demo-pinecone/embed/embed_elementweb.go b/cmd/dendrite-demo-pinecone/embed/embed_elementweb.go index 8b3be72c1..d37362e21 100644 --- a/cmd/dendrite-demo-pinecone/embed/embed_elementweb.go +++ b/cmd/dendrite-demo-pinecone/embed/embed_elementweb.go @@ -1,3 +1,17 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build elementweb // +build elementweb diff --git a/cmd/dendrite-demo-pinecone/embed/embed_other.go b/cmd/dendrite-demo-pinecone/embed/embed_other.go index a4b223452..94360fce6 100644 --- a/cmd/dendrite-demo-pinecone/embed/embed_other.go +++ b/cmd/dendrite-demo-pinecone/embed/embed_other.go @@ -1,3 +1,17 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build !elementweb // +build !elementweb diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index a6abf06e5..122da1c54 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. +// Copyright 2022 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -245,6 +245,7 @@ func main() { logrus.WithError(err).Error("Failed to connect WebSocket peer to Pinecone switch") } }) + httpRouter.HandleFunc("/pinecone", pRouter.ManholeHandler) embed.Embed(httpRouter, *instancePort, "Pinecone Demo") pMux := mux.NewRouter().SkipClean(true).UseEncodedPath() diff --git a/cmd/dendrite-demo-pinecone/rooms/rooms.go b/cmd/dendrite-demo-pinecone/rooms/rooms.go index 5972d129f..0fafbedc3 100644 --- a/cmd/dendrite-demo-pinecone/rooms/rooms.go +++ b/cmd/dendrite-demo-pinecone/rooms/rooms.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. +// Copyright 2022 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ import ( "sync" "time" + "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/defaults" "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -50,9 +51,12 @@ func NewPineconeRoomProvider( } func (p *PineconeRoomProvider) Rooms() []gomatrixserverlib.PublicRoom { - list := []gomatrixserverlib.ServerName{} + list := map[gomatrixserverlib.ServerName]struct{}{} + for k := range defaults.DefaultServerNames { + list[k] = struct{}{} + } for _, k := range p.r.Peers() { - list = append(list, gomatrixserverlib.ServerName(k.PublicKey)) + list[gomatrixserverlib.ServerName(k.PublicKey)] = struct{}{} } return bulkFetchPublicRoomsFromServers(context.Background(), p.fedClient, list) } @@ -61,7 +65,7 @@ func (p *PineconeRoomProvider) Rooms() []gomatrixserverlib.PublicRoom { // Returns a list of public rooms. func bulkFetchPublicRoomsFromServers( ctx context.Context, fedClient *gomatrixserverlib.FederationClient, - homeservers []gomatrixserverlib.ServerName, + homeservers map[gomatrixserverlib.ServerName]struct{}, ) (publicRooms []gomatrixserverlib.PublicRoom) { limit := 200 // follow pipeline semantics, see https://blog.golang.org/pipelines for more info. @@ -74,7 +78,7 @@ func bulkFetchPublicRoomsFromServers( wg.Add(len(homeservers)) // concurrently query for public rooms reqctx, reqcancel := context.WithTimeout(ctx, time.Second*5) - for _, hs := range homeservers { + for hs := range homeservers { go func(homeserverDomain gomatrixserverlib.ServerName) { defer wg.Done() util.GetLogger(reqctx).WithField("hs", homeserverDomain).Info("Querying HS for public rooms") diff --git a/cmd/dendrite-demo-pinecone/users/users.go b/cmd/dendrite-demo-pinecone/users/users.go index ffbd27ee9..ebfb5cbe3 100644 --- a/cmd/dendrite-demo-pinecone/users/users.go +++ b/cmd/dendrite-demo-pinecone/users/users.go @@ -1,3 +1,17 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package users import ( @@ -11,6 +25,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" clienthttputil "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/defaults" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -64,9 +79,12 @@ func (p *PineconeUserProvider) FederatedUserProfiles(w http.ResponseWriter, r *h } func (p *PineconeUserProvider) QuerySearchProfiles(ctx context.Context, req *userapi.QuerySearchProfilesRequest, res *userapi.QuerySearchProfilesResponse) error { - list := map[string]struct{}{} + list := map[gomatrixserverlib.ServerName]struct{}{} + for k := range defaults.DefaultServerNames { + list[k] = struct{}{} + } for _, k := range p.r.Peers() { - list[k.PublicKey] = struct{}{} + list[gomatrixserverlib.ServerName(k.PublicKey)] = struct{}{} } res.Profiles = bulkFetchUserDirectoriesFromServers(context.Background(), req, p.fedClient, list) return nil @@ -77,7 +95,7 @@ func (p *PineconeUserProvider) QuerySearchProfiles(ctx context.Context, req *use func bulkFetchUserDirectoriesFromServers( ctx context.Context, req *userapi.QuerySearchProfilesRequest, fedClient *gomatrixserverlib.FederationClient, - homeservers map[string]struct{}, + homeservers map[gomatrixserverlib.ServerName]struct{}, ) (profiles []authtypes.Profile) { jsonBody, err := json.Marshal(req) if err != nil { @@ -96,7 +114,7 @@ func bulkFetchUserDirectoriesFromServers( // concurrently query for public rooms reqctx, reqcancel := context.WithTimeout(ctx, time.Second*5) for hs := range homeservers { - go func(homeserverDomain string) { + go func(homeserverDomain gomatrixserverlib.ServerName) { defer wg.Done() util.GetLogger(reqctx).WithField("hs", homeserverDomain).Info("Querying HS for users") @@ -115,7 +133,7 @@ func bulkFetchUserDirectoriesFromServers( return } for _, profile := range res.Profiles { - profile.ServerName = homeserverDomain + profile.ServerName = string(homeserverDomain) // atomically send a room or stop select { case profileCh <- profile: diff --git a/go.mod b/go.mod index ae925c719..8baa44d3c 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 github.com/matrix-org/gomatrixserverlib v0.0.0-20220317164600-0980b7f341e0 - github.com/matrix-org/pinecone v0.0.0-20220323142759-6fb077377278 + github.com/matrix-org/pinecone v0.0.0-20220330132624-fb51a311e4b8 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.10 github.com/morikuni/aec v1.0.0 // indirect diff --git a/go.sum b/go.sum index 25174f468..92fb70bde 100644 --- a/go.sum +++ b/go.sum @@ -943,8 +943,8 @@ github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= github.com/matrix-org/gomatrixserverlib v0.0.0-20220317164600-0980b7f341e0 h1:IINbE/0jSYGb7M31StazufyIQdYWSivRlhuns3JYPOM= github.com/matrix-org/gomatrixserverlib v0.0.0-20220317164600-0980b7f341e0/go.mod h1:+WF5InseAMgi1fTnU46JH39IDpEvLep0fDzx9LDf2Bo= -github.com/matrix-org/pinecone v0.0.0-20220323142759-6fb077377278 h1:lRrvMMv7x1FIVW1mcBdU89lvbgAXKz6RyYR0VQTAr3E= -github.com/matrix-org/pinecone v0.0.0-20220323142759-6fb077377278/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk= +github.com/matrix-org/pinecone v0.0.0-20220330132624-fb51a311e4b8 h1:nMRg2106UjC98dAW236yJp9gavS4247fRftY/avkNZY= +github.com/matrix-org/pinecone v0.0.0-20220330132624-fb51a311e4b8/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From cd8fac152eb2ea9165646e08bd87e3e95bd4706a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 1 Apr 2022 16:14:38 +0100 Subject: [PATCH 40/55] Include joined and invite member counts in room summary (#2315) * Include joined and invite member counts in room summary This should fix #2314 and also fix the problem where some clients like Element Android, Fluffychat etc would display the wrong member count for a given room. * Improve SQLite query precision * Check existence of state key for membership events --- syncapi/storage/interface.go | 1 + syncapi/storage/postgres/memberships_table.go | 21 ++++++++++++++++-- syncapi/storage/shared/syncserver.go | 4 ++++ syncapi/storage/sqlite3/memberships_table.go | 21 ++++++++++++++++-- syncapi/storage/tables/interface.go | 1 + syncapi/streams/stream_pdu.go | 22 +++++++++++++++++++ syncapi/types/types.go | 5 +++++ sytest-whitelist | 1 + 8 files changed, 72 insertions(+), 4 deletions(-) diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 647fffad5..03313ec6e 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -37,6 +37,7 @@ type Database interface { GetStateDeltasForFullStateSync(ctx context.Context, device *userapi.Device, r types.Range, userID string, stateFilter *gomatrixserverlib.StateFilter) ([]types.StateDelta, []string, error) GetStateDeltas(ctx context.Context, device *userapi.Device, r types.Range, userID string, stateFilter *gomatrixserverlib.StateFilter) ([]types.StateDelta, []string, error) RoomIDsWithMembership(ctx context.Context, userID string, membership string) ([]string, error) + MembershipCount(ctx context.Context, roomID, membership string, pos types.StreamPosition) (int, error) RecentEvents(ctx context.Context, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error) diff --git a/syncapi/storage/postgres/memberships_table.go b/syncapi/storage/postgres/memberships_table.go index 6566544d6..1242a3221 100644 --- a/syncapi/storage/postgres/memberships_table.go +++ b/syncapi/storage/postgres/memberships_table.go @@ -62,9 +62,15 @@ const selectMembershipSQL = "" + " ORDER BY stream_pos DESC" + " LIMIT 1" +const selectMembershipCountSQL = "" + + "SELECT COUNT(*) FROM (" + + " SELECT DISTINCT ON (room_id, user_id) room_id, user_id, membership FROM syncapi_memberships WHERE room_id = $1 AND stream_pos <= $2 ORDER BY room_id, user_id, stream_pos DESC" + + ") t WHERE t.membership = $3" + type membershipsStatements struct { - upsertMembershipStmt *sql.Stmt - selectMembershipStmt *sql.Stmt + upsertMembershipStmt *sql.Stmt + selectMembershipStmt *sql.Stmt + selectMembershipCountStmt *sql.Stmt } func NewPostgresMembershipsTable(db *sql.DB) (tables.Memberships, error) { @@ -79,6 +85,9 @@ func NewPostgresMembershipsTable(db *sql.DB) (tables.Memberships, error) { if s.selectMembershipStmt, err = db.Prepare(selectMembershipSQL); err != nil { return nil, err } + if s.selectMembershipCountStmt, err = db.Prepare(selectMembershipCountSQL); err != nil { + return nil, err + } return s, nil } @@ -109,3 +118,11 @@ func (s *membershipsStatements) SelectMembership( err = stmt.QueryRowContext(ctx, roomID, userID, memberships).Scan(&eventID, &streamPos, &topologyPos) return } + +func (s *membershipsStatements) SelectMembershipCount( + ctx context.Context, txn *sql.Tx, roomID, membership string, pos types.StreamPosition, +) (count int, err error) { + stmt := sqlutil.TxStmt(txn, s.selectMembershipCountStmt) + err = stmt.QueryRowContext(ctx, roomID, pos, membership).Scan(&count) + return +} diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index 349e44526..de43678d7 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -118,6 +118,10 @@ func (d *Database) RoomIDsWithMembership(ctx context.Context, userID string, mem return d.CurrentRoomState.SelectRoomIDsWithMembership(ctx, nil, userID, membership) } +func (d *Database) MembershipCount(ctx context.Context, roomID, membership string, pos types.StreamPosition) (int, error) { + return d.Memberships.SelectMembershipCount(ctx, nil, roomID, membership, pos) +} + func (d *Database) RecentEvents(ctx context.Context, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error) { return d.OutputEvents.SelectRecentEvents(ctx, nil, roomID, r, eventFilter, chronologicalOrder, onlySyncEvents) } diff --git a/syncapi/storage/sqlite3/memberships_table.go b/syncapi/storage/sqlite3/memberships_table.go index e5445e815..776bf3da3 100644 --- a/syncapi/storage/sqlite3/memberships_table.go +++ b/syncapi/storage/sqlite3/memberships_table.go @@ -63,9 +63,15 @@ const selectMembershipSQL = "" + " ORDER BY stream_pos DESC" + " LIMIT 1" +const selectMembershipCountSQL = "" + + "SELECT COUNT(*) FROM (" + + " SELECT * FROM syncapi_memberships WHERE room_id = $1 AND stream_pos <= $2 GROUP BY user_id HAVING(max(stream_pos))" + + ") t WHERE t.membership = $3" + type membershipsStatements struct { - db *sql.DB - upsertMembershipStmt *sql.Stmt + db *sql.DB + upsertMembershipStmt *sql.Stmt + selectMembershipCountStmt *sql.Stmt } func NewSqliteMembershipsTable(db *sql.DB) (tables.Memberships, error) { @@ -79,6 +85,9 @@ func NewSqliteMembershipsTable(db *sql.DB) (tables.Memberships, error) { if s.upsertMembershipStmt, err = db.Prepare(upsertMembershipSQL); err != nil { return nil, err } + if s.selectMembershipCountStmt, err = db.Prepare(selectMembershipCountSQL); err != nil { + return nil, err + } return s, nil } @@ -117,3 +126,11 @@ func (s *membershipsStatements) SelectMembership( err = sqlutil.TxStmt(txn, stmt).QueryRowContext(ctx, params...).Scan(&eventID, &streamPos, &topologyPos) return } + +func (s *membershipsStatements) SelectMembershipCount( + ctx context.Context, txn *sql.Tx, roomID, membership string, pos types.StreamPosition, +) (count int, err error) { + stmt := sqlutil.TxStmt(txn, s.selectMembershipCountStmt) + err = stmt.QueryRowContext(ctx, roomID, pos, membership).Scan(&count) + return +} diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index ba0076e22..2c29888d3 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -174,6 +174,7 @@ type Receipts interface { type Memberships interface { UpsertMembership(ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, streamPos, topologicalPos types.StreamPosition) error SelectMembership(ctx context.Context, txn *sql.Tx, roomID, userID, memberships []string) (eventID string, streamPos, topologyPos types.StreamPosition, err error) + SelectMembershipCount(ctx context.Context, txn *sql.Tx, roomID, membership string, pos types.StreamPosition) (count int, err error) } type NotificationData interface { diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index ccdac0864..d23209af3 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -253,9 +253,25 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( updateLatestPosition(delta.StateEvents[len(delta.StateEvents)-1].EventID()) } + hasMembershipChange := false + for _, recentEvent := range recentStreamEvents { + if recentEvent.Type() == gomatrixserverlib.MRoomMember && recentEvent.StateKey() != nil { + hasMembershipChange = true + break + } + } + + // Work out how many members are in the room. + joinedCount, _ := p.DB.MembershipCount(ctx, delta.RoomID, gomatrixserverlib.Join, latestPosition) + invitedCount, _ := p.DB.MembershipCount(ctx, delta.RoomID, gomatrixserverlib.Invite, latestPosition) + switch delta.Membership { case gomatrixserverlib.Join: jr := types.NewJoinResponse() + if hasMembershipChange { + jr.Summary.JoinedMemberCount = &joinedCount + jr.Summary.InvitedMemberCount = &invitedCount + } jr.Timeline.PrevBatch = &prevBatch jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = limited @@ -367,12 +383,18 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( prevBatch.Decrement() } + // Work out how many members are in the room. + joinedCount, _ := p.DB.MembershipCount(ctx, roomID, gomatrixserverlib.Join, r.From) + invitedCount, _ := p.DB.MembershipCount(ctx, roomID, gomatrixserverlib.Invite, r.From) + // We don't include a device here as we don't need to send down // transaction IDs for complete syncs, but we do it anyway because Sytest demands it for: // "Can sync a room with a message with a transaction id" - which does a complete sync to check. recentEvents := p.DB.StreamEventsToEvents(device, recentStreamEvents) stateEvents = removeDuplicates(stateEvents, recentEvents) jr = types.NewJoinResponse() + jr.Summary.JoinedMemberCount = &joinedCount + jr.Summary.InvitedMemberCount = &invitedCount jr.Timeline.PrevBatch = prevBatch jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = limited diff --git a/syncapi/types/types.go b/syncapi/types/types.go index f964b80b5..d0efa1bbb 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -377,6 +377,11 @@ func (r *Response) IsEmpty() bool { // JoinResponse represents a /sync response for a room which is under the 'join' or 'peek' key. type JoinResponse struct { + Summary struct { + Heroes []string `json:"m.heroes,omitempty"` + JoinedMemberCount *int `json:"m.joined_member_count,omitempty"` + InvitedMemberCount *int `json:"m.invited_member_count,omitempty"` + } `json:"summary"` State struct { Events []gomatrixserverlib.ClientEvent `json:"events"` } `json:"state"` diff --git a/sytest-whitelist b/sytest-whitelist index 40bf5afac..7614fbb96 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -661,3 +661,4 @@ Canonical alias can include alt_aliases Can delete canonical alias AS can make room aliases /context/ with lazy_load_members filter works +Room summary counts change when membership changes From a0f5d8e1a503b3932b07e0786a92c64a9bbda88a Mon Sep 17 00:00:00 2001 From: S7evinK <2353100+S7evinK@users.noreply.github.com> Date: Mon, 4 Apr 2022 10:32:53 +0200 Subject: [PATCH 41/55] Fix unlimitedSize uploads (#2317) --- mediaapi/routing/upload.go | 2 +- mediaapi/routing/upload_test.go | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/mediaapi/routing/upload.go b/mediaapi/routing/upload.go index ecdab2195..f762b2ff5 100644 --- a/mediaapi/routing/upload.go +++ b/mediaapi/routing/upload.go @@ -169,7 +169,7 @@ func (r *uploadRequest) doUpload( } // Check if temp file size exceeds max file size configuration - if bytesWritten > types.FileSizeBytes(*cfg.MaxFileSizeBytes) { + if *cfg.MaxFileSizeBytes > 0 && bytesWritten > types.FileSizeBytes(*cfg.MaxFileSizeBytes) { fileutils.RemoveDir(tmpDir, r.Logger) // delete temp file return requestEntityTooLargeJSONResponse(*cfg.MaxFileSizeBytes) } diff --git a/mediaapi/routing/upload_test.go b/mediaapi/routing/upload_test.go index 032437b59..e81254f35 100644 --- a/mediaapi/routing/upload_test.go +++ b/mediaapi/routing/upload_test.go @@ -36,6 +36,7 @@ func Test_uploadRequest_doUpload(t *testing.T) { } maxSize := config.FileSizeBytes(8) + unlimitedSize := config.FileSizeBytes(0) logger := log.New().WithField("mediaapi", "test") testdataPath := filepath.Join(wd, "./testdata") @@ -117,6 +118,27 @@ func Test_uploadRequest_doUpload(t *testing.T) { }, want: requestEntityTooLargeJSONResponse(maxSize), }, + { + name: "upload ok with unlimited filesize", + args: args{ + ctx: context.Background(), + reqReader: strings.NewReader("test test test"), + cfg: &config.MediaAPI{ + MaxFileSizeBytes: &unlimitedSize, + BasePath: config.Path(testdataPath), + AbsBasePath: config.Path(testdataPath), + DynamicThumbnails: false, + }, + db: db, + }, + fields: fields{ + Logger: logger, + MediaMetadata: &types.MediaMetadata{ + MediaID: "1339", + UploadName: "test fail", + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 6748a2a823fe5923368371ef7a109d856a177441 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 4 Apr 2022 11:02:16 +0100 Subject: [PATCH 42/55] Fix sign-off link in `CONTRIBUTING.md` --- docs/CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index fe7127c76..116adfae6 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -94,4 +94,4 @@ For more general questions please use We ask that everyone who contributes to the project signs off their contributions, in accordance with the -[DCO](https://github.com/matrix-org/matrix-doc/blob/main/CONTRIBUTING.rst#sign-off). +[DCO](https://github.com/matrix-org/matrix-spec/blob/main/CONTRIBUTING.rst#sign-off). From 9b316ac64ce4ff0abee497e9c69204c4eb9102eb Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 4 Apr 2022 15:14:10 +0100 Subject: [PATCH 43/55] Slower federation warm-up (#2320) * Wake destination queues gradually, rather than all at once * Delay device list updates too * Maximum two minute warmup period --- federationapi/queue/queue.go | 41 +++++++++++++----------- keyserver/internal/device_list_update.go | 9 +++++- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go index dcd090856..5b5481274 100644 --- a/federationapi/queue/queue.go +++ b/federationapi/queue/queue.go @@ -104,28 +104,31 @@ func NewOutgoingQueues( } // Look up which servers we have pending items for and then rehydrate those queues. if !disabled { - time.AfterFunc(time.Second*5, func() { - serverNames := map[gomatrixserverlib.ServerName]struct{}{} - if names, err := db.GetPendingPDUServerNames(context.Background()); err == nil { - for _, serverName := range names { - serverNames[serverName] = struct{}{} - } - } else { - log.WithError(err).Error("Failed to get PDU server names for destination queue hydration") + serverNames := map[gomatrixserverlib.ServerName]struct{}{} + if names, err := db.GetPendingPDUServerNames(context.Background()); err == nil { + for _, serverName := range names { + serverNames[serverName] = struct{}{} } - if names, err := db.GetPendingEDUServerNames(context.Background()); err == nil { - for _, serverName := range names { - serverNames[serverName] = struct{}{} - } - } else { - log.WithError(err).Error("Failed to get EDU server names for destination queue hydration") + } else { + log.WithError(err).Error("Failed to get PDU server names for destination queue hydration") + } + if names, err := db.GetPendingEDUServerNames(context.Background()); err == nil { + for _, serverName := range names { + serverNames[serverName] = struct{}{} } - for serverName := range serverNames { - if queue := queues.getQueue(serverName); queue != nil { - queue.wakeQueueIfNeeded() - } + } else { + log.WithError(err).Error("Failed to get EDU server names for destination queue hydration") + } + offset, step := time.Second*5, time.Second + if max := len(serverNames); max > 120 { + step = (time.Second * 120) / time.Duration(max) + } + for serverName := range serverNames { + if queue := queues.getQueue(serverName); queue != nil { + time.AfterFunc(offset, queue.wakeQueueIfNeeded) + offset += step } - }) + } } return queues } diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go index 4b2b8c187..561c9a163 100644 --- a/keyserver/internal/device_list_update.go +++ b/keyserver/internal/device_list_update.go @@ -157,8 +157,15 @@ func (u *DeviceListUpdater) Start() error { if err != nil { return err } + offset, step := time.Second*10, time.Second + if max := len(staleLists); max > 120 { + step = (time.Second * 120) / time.Duration(max) + } for _, userID := range staleLists { - u.notifyWorkers(userID) + time.AfterFunc(offset, func() { + u.notifyWorkers(userID) + }) + offset += step } return nil } From c69159bda751afe035d33ddd13ae40e0283988b3 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 4 Apr 2022 15:16:02 +0100 Subject: [PATCH 44/55] Update to matrix-org/pinecone@e526fa8 --- build/gobind-pinecone/monolith.go | 14 ++- cmd/dendrite-demo-pinecone/conn/client.go | 17 ++-- cmd/dendrite-demo-pinecone/main.go | 12 +-- go.mod | 6 +- go.sum | 118 ++++++++++++++++++++-- 5 files changed, 134 insertions(+), 33 deletions(-) diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 97ce08d82..90d572358 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -23,7 +23,6 @@ import ( "fmt" "io" "io/ioutil" - "log" "net" "net/http" "os" @@ -93,7 +92,7 @@ func (m *DendriteMonolith) PeerCount(peertype int) int { } func (m *DendriteMonolith) SessionCount() int { - return len(m.PineconeQUIC.Sessions()) + return len(m.PineconeQUIC.Protocol("matrix").Sessions()) } func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) { @@ -283,10 +282,9 @@ func (m *DendriteMonolith) Start() { m.logger.SetOutput(BindLogger{}) logrus.SetOutput(BindLogger{}) - logger := log.New(os.Stdout, "PINECONE: ", 0) - m.PineconeRouter = pineconeRouter.NewRouter(logger, sk, false) - m.PineconeQUIC = pineconeSessions.NewSessions(logger, m.PineconeRouter) - m.PineconeMulticast = pineconeMulticast.NewMulticast(logger, m.PineconeRouter) + m.PineconeRouter = pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk, false) + m.PineconeQUIC = pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), m.PineconeRouter, []string{"matrix"}) + m.PineconeMulticast = pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), m.PineconeRouter) prefix := hex.EncodeToString(pk) cfg := &config.Dendrite{} @@ -374,7 +372,7 @@ func (m *DendriteMonolith) Start() { pMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux) pMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux) - pHTTP := m.PineconeQUIC.HTTP() + pHTTP := m.PineconeQUIC.Protocol("matrix").HTTP() pHTTP.Mux().Handle(users.PublicURL, pMux) pHTTP.Mux().Handle(httputil.PublicFederationPathPrefix, pMux) pHTTP.Mux().Handle(httputil.PublicMediaPathPrefix, pMux) @@ -400,7 +398,7 @@ func (m *DendriteMonolith) Start() { go func() { m.logger.Info("Listening on ", cfg.Global.ServerName) - m.logger.Fatal(m.httpServer.Serve(m.PineconeQUIC)) + m.logger.Fatal(m.httpServer.Serve(m.PineconeQUIC.Protocol("matrix"))) }() go func() { logrus.Info("Listening on ", m.listener.Addr()) diff --git a/cmd/dendrite-demo-pinecone/conn/client.go b/cmd/dendrite-demo-pinecone/conn/client.go index 59de9690c..27e18c2a3 100644 --- a/cmd/dendrite-demo-pinecone/conn/client.go +++ b/cmd/dendrite-demo-pinecone/conn/client.go @@ -67,21 +67,22 @@ func (y *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { } func createTransport(s *pineconeSessions.Sessions) *http.Transport { + proto := s.Protocol("matrix") tr := &http.Transport{ DisableKeepAlives: false, - Dial: s.Dial, - DialContext: s.DialContext, - DialTLS: s.DialTLS, - DialTLSContext: s.DialTLSContext, + Dial: proto.Dial, + DialContext: proto.DialContext, + DialTLS: proto.DialTLS, + DialTLSContext: proto.DialTLSContext, } tr.RegisterProtocol( "matrix", &RoundTripper{ inner: &http.Transport{ DisableKeepAlives: false, - Dial: s.Dial, - DialContext: s.DialContext, - DialTLS: s.DialTLS, - DialTLSContext: s.DialTLSContext, + Dial: proto.Dial, + DialContext: proto.DialContext, + DialTLS: proto.DialTLS, + DialTLSContext: proto.DialTLSContext, }, }, ) diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index 122da1c54..5c086b7ef 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -22,7 +22,6 @@ import ( "flag" "fmt" "io/ioutil" - "log" "net" "net/http" "os" @@ -91,8 +90,7 @@ func main() { pk = sk.Public().(ed25519.PublicKey) } - logger := log.New(os.Stdout, "", 0) - pRouter := pineconeRouter.NewRouter(logger, sk, false) + pRouter := pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk, false) go func() { listener, err := net.Listen("tcp", *instanceListen) @@ -122,8 +120,8 @@ func main() { } }() - pQUIC := pineconeSessions.NewSessions(logger, pRouter) - pMulticast := pineconeMulticast.NewMulticast(logger, pRouter) + pQUIC := pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), pRouter, []string{"matrix"}) + pMulticast := pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), pRouter) pMulticast.Start() connectToStaticPeer := func() { @@ -253,7 +251,7 @@ func main() { pMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux) pMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux) - pHTTP := pQUIC.HTTP() + pHTTP := pQUIC.Protocol("matrix").HTTP() pHTTP.Mux().Handle(users.PublicURL, pMux) pHTTP.Mux().Handle(httputil.PublicFederationPathPrefix, pMux) pHTTP.Mux().Handle(httputil.PublicMediaPathPrefix, pMux) @@ -275,7 +273,7 @@ func main() { go func() { pubkey := pRouter.PublicKey() logrus.Info("Listening on ", hex.EncodeToString(pubkey[:])) - logrus.Fatal(httpServer.Serve(pQUIC)) + logrus.Fatal(httpServer.Serve(pQUIC.Protocol("matrix"))) }() go func() { httpBindAddr := fmt.Sprintf(":%d", *instancePort) diff --git a/go.mod b/go.mod index 8baa44d3c..4830bf648 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 github.com/matrix-org/gomatrixserverlib v0.0.0-20220317164600-0980b7f341e0 - github.com/matrix-org/pinecone v0.0.0-20220330132624-fb51a311e4b8 + github.com/matrix-org/pinecone v0.0.0-20220404141326-e526fa82f79d github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.10 github.com/morikuni/aec v1.0.0 // indirect @@ -48,8 +48,6 @@ require ( github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 github.com/ngrok/sqlmw v0.0.0-20211220175533-9d16fdc47b31 - github.com/onsi/ginkgo v1.16.4 // indirect - github.com/onsi/gomega v1.13.0 // indirect github.com/opentracing/opentracing-go v1.2.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 @@ -64,7 +62,7 @@ require ( go.uber.org/atomic v1.9.0 golang.org/x/crypto v0.0.0-20220214200702-86341886e292 golang.org/x/image v0.0.0-20211028202545-6944b10bf410 - golang.org/x/mobile v0.0.0-20220112015953-858099ff7816 + golang.org/x/mobile v0.0.0-20220325161704-447654d348e3 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 diff --git a/go.sum b/go.sum index 92fb70bde..ae9f0fb7b 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= @@ -31,7 +33,12 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= +dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= +dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= +git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/Arceliar/ironwood v0.0.0-20211125050254-8951369625d0 h1:QUqcb7BOcBU2p7Nax7pESOb8hrZYtI0Ts6j4v4mvcQo= @@ -117,6 +124,7 @@ github.com/anacrolix/missinggo v1.2.1 h1:0IE3TqX5y5D0IxeMwTyIgqdDew4QrzcXaaEnJQy github.com/anacrolix/missinggo v1.2.1/go.mod h1:J5cMhif8jPmFoC3+Uvob3OXXNIhOUikzMt+uUjeM21Y= github.com/anacrolix/missinggo/perf v1.0.0/go.mod h1:ljAFWkBuzkO12MQclXzZrosP5urunoLS0Cbvb4V0uMQ= github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= @@ -135,6 +143,7 @@ github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edY github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo= github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo= github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8= @@ -154,6 +163,7 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= @@ -166,6 +176,8 @@ github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cb github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= +github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= +github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -276,6 +288,7 @@ github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmf github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= @@ -345,10 +358,12 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k= github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= @@ -370,6 +385,7 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= +github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= @@ -403,6 +419,7 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+ github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= @@ -433,6 +450,7 @@ github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -440,6 +458,8 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -478,6 +498,7 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -499,6 +520,8 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= @@ -517,6 +540,7 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= @@ -541,9 +565,8 @@ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hjson/hjson-go v3.1.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk= github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= -github.com/huandu/xstrings v1.2.0 h1:yPeWdRnmynF7p+lLYz0H2tthW9lqhMJrQV/U7yy4wX0= -github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= @@ -618,6 +641,7 @@ github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsj github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= +github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -700,6 +724,7 @@ github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -919,15 +944,26 @@ github.com/libp2p/go-yamux v1.4.1 h1:P1Fe9vF4th5JOxxgQvfbOHkrGqIZniTLf+ddhZp8YTI github.com/libp2p/go-yamux v1.4.1/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= github.com/libp2p/go-yamux/v2 v2.0.0 h1:vSGhAy5u6iHBq11ZDcyHH4Blcf9xlBhT4WQDoOE90LU= github.com/libp2p/go-yamux/v2 v2.0.0/go.mod h1:NVWira5+sVUIU6tu1JWvaRn1dRnG+cawOJiflsAM+7U= +github.com/lucas-clemente/quic-go v0.26.0 h1:ALBQXr9UJ8A1LyzvceX4jd9QFsHvlI0RR6BkV16o00A= +github.com/lucas-clemente/quic-go v0.26.0/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI= github.com/lunixbochs/vtclean v0.0.0-20160125035106-4fbf7632a2c6/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= +github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lxn/walk v0.0.0-20210112085537-c389da54e794/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ= github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= +github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= +github.com/marten-seemann/qtls-go1-16 v0.1.5 h1:o9JrYPPco/Nukd/HpOHMHZoBDXQqoNtUCmny98/1uqQ= +github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk= +github.com/marten-seemann/qtls-go1-17 v0.1.1 h1:DQjHPq+aOzUeh9/lixAGunn6rIOQyWChPSI4+hgW7jc= +github.com/marten-seemann/qtls-go1-17 v0.1.1/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s= +github.com/marten-seemann/qtls-go1-18 v0.1.1 h1:qp7p7XXUFL7fpBvSS1sWD+uSqPvzNQK43DH+/qEkj0Y= +github.com/marten-seemann/qtls-go1-18 v0.1.1/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= github.com/masterzen/azure-sdk-for-go v3.2.0-beta.0.20161014135628-ee4f0065d00c+incompatible/go.mod h1:mf8fjOu33zCqxUjuiU3I8S1lJMyEAlH+0F2+M5xl3hE= github.com/masterzen/simplexml v0.0.0-20160608183007-4572e39b1ab9/go.mod h1:kCEbxUJlNDEBNbdQMkPSp6yaKcRXVI6f4ddk8Riv4bc= github.com/masterzen/winrm v0.0.0-20161014151040-7a535cd943fc/go.mod h1:CfZSN7zwz5gJiFhZJz49Uzk7mEBHIceWmbFmYx7Hf7E= @@ -943,8 +979,8 @@ github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= github.com/matrix-org/gomatrixserverlib v0.0.0-20220317164600-0980b7f341e0 h1:IINbE/0jSYGb7M31StazufyIQdYWSivRlhuns3JYPOM= github.com/matrix-org/gomatrixserverlib v0.0.0-20220317164600-0980b7f341e0/go.mod h1:+WF5InseAMgi1fTnU46JH39IDpEvLep0fDzx9LDf2Bo= -github.com/matrix-org/pinecone v0.0.0-20220330132624-fb51a311e4b8 h1:nMRg2106UjC98dAW236yJp9gavS4247fRftY/avkNZY= -github.com/matrix-org/pinecone v0.0.0-20220330132624-fb51a311e4b8/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk= +github.com/matrix-org/pinecone v0.0.0-20220404141326-e526fa82f79d h1:1+T4eOPRsf6cr0lMPW4oO2k8TTHm4mqIh65kpEID5Rk= +github.com/matrix-org/pinecone v0.0.0-20220404141326-e526fa82f79d/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= @@ -977,6 +1013,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182aff github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.25/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= @@ -1087,11 +1124,12 @@ github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= +github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= +github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/neilalexander/nats-server/v2 v2.7.5-0.20220311134712-e2e4a244f30e h1:5tEHLzvDeS6IeqO2o9FFhsE3V2erYj8FlMt2J91wzsk= github.com/neilalexander/nats-server/v2 v2.7.5-0.20220311134712-e2e4a244f30e/go.mod h1:1vZ2Nijh8tcyNe8BDVyTviCd9NYzRbubQYiEHsvOQWc= github.com/neilalexander/nats.go v1.11.1-0.20220104162523-f4ddebe1061c h1:G2qsv7D0rY94HAu8pXmElMluuMHQ85waxIDQBhIzV2Q= github.com/neilalexander/nats.go v1.11.1-0.20220104162523-f4ddebe1061c/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= -github.com/neilalexander/utp v0.1.1-0.20210622132614-ee9a34a30488/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8= github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 h1:lrVQzBtkeQEGGYUHwSX1XPe1E5GL6U3KYCNe2G4bncQ= github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= @@ -1115,6 +1153,7 @@ github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= @@ -1159,6 +1198,7 @@ github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -1178,6 +1218,7 @@ github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prY github.com/pressly/goose v2.7.0+incompatible h1:PWejVEv07LCerQEzMMeAtjuyCKbyprZ/LBa6K5P0OCQ= github.com/pressly/goose v2.7.0+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8= github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -1193,6 +1234,7 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -1202,6 +1244,7 @@ github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9 github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -1230,7 +1273,29 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= +github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= +github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= +github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= +github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= +github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI= +github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU= +github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag= +github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg= +github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw= +github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y= +github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= +github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ= +github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I= +github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0= +github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ= +github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk= +github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= +github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -1246,6 +1311,8 @@ github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:s github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= +github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= @@ -1287,6 +1354,7 @@ github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w= @@ -1321,6 +1389,8 @@ github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBn github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= +github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= @@ -1359,6 +1429,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= @@ -1368,6 +1439,7 @@ go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= +go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= @@ -1393,16 +1465,20 @@ go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= +golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180214000028-650f4a345ab4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -1445,6 +1521,7 @@ golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMx golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ= golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1458,8 +1535,9 @@ golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mobile v0.0.0-20220112015953-858099ff7816 h1:jhDgkcu3yQ4tasBZ+1YwDmK7eFmuVf1w1k+NGGGxfmE= golang.org/x/mobile v0.0.0-20220112015953-858099ff7816/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ= +golang.org/x/mobile v0.0.0-20220325161704-447654d348e3 h1:ZDL7hDvJEQEcHVkoZawKmRUgbqn1pOIzb8EinBh5csU= +golang.org/x/mobile v0.0.0-20220325161704-447654d348e3/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -1473,12 +1551,15 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -1527,11 +1608,14 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1547,6 +1631,7 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1556,6 +1641,7 @@ golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1604,6 +1690,7 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1629,6 +1716,7 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1664,7 +1752,9 @@ golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 h1:GZokNIeuVkl3aZHJchRrr13W golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1720,6 +1810,7 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098 h1:YuekqPskqwCCPM79F1X5Dhv4ezTCj+Ki1oNwiafxkA0= golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= @@ -1737,6 +1828,9 @@ gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1754,6 +1848,8 @@ google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= @@ -1762,6 +1858,9 @@ google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= +google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1795,6 +1894,9 @@ google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a h1:pOwg4OoaRYScjmR4LlLgdtnyoHYTSAVhhqe5uPdpII8= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1888,6 +1990,8 @@ gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1935,3 +2039,5 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= +sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= From cee12a7ab06c0b41499e13ccd8d4ea3cd4832ab0 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 4 Apr 2022 15:35:47 +0100 Subject: [PATCH 45/55] Enhanced calm at startup of Pinecone demos --- build/gobind-pinecone/monolith.go | 11 +---------- cmd/dendrite-demo-pinecone/main.go | 9 --------- 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 90d572358..608599498 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -38,7 +38,6 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/users" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" "github.com/matrix-org/dendrite/federationapi" - "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/roomserver" @@ -271,7 +270,7 @@ func (m *DendriteMonolith) Start() { pk = sk.Public().(ed25519.PublicKey) } - m.listener, err = net.Listen("tcp", "localhost:65432") + m.listener, err = net.Listen("tcp", ":65432") if err != nil { panic(err) } @@ -404,14 +403,6 @@ func (m *DendriteMonolith) Start() { logrus.Info("Listening on ", m.listener.Addr()) logrus.Fatal(http.Serve(m.listener, httpRouter)) }() - go func() { - logrus.Info("Sending wake-up message to known nodes") - req := &api.PerformBroadcastEDURequest{} - res := &api.PerformBroadcastEDUResponse{} - if err := fsAPI.PerformBroadcastEDU(context.TODO(), req, res); err != nil { - logrus.WithError(err).Error("Failed to send wake-up message to known nodes") - } - }() } func (m *DendriteMonolith) Stop() { diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index 5c086b7ef..a3d3ed175 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -37,7 +37,6 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/users" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" "github.com/matrix-org/dendrite/federationapi" - "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/keyserver" @@ -280,14 +279,6 @@ func main() { logrus.Info("Listening on ", httpBindAddr) logrus.Fatal(http.ListenAndServe(httpBindAddr, httpRouter)) }() - go func() { - logrus.Info("Sending wake-up message to known nodes") - req := &api.PerformBroadcastEDURequest{} - res := &api.PerformBroadcastEDUResponse{} - if err := fsAPI.PerformBroadcastEDU(context.TODO(), req, res); err != nil { - logrus.WithError(err).Error("Failed to send wake-up message to known nodes") - } - }() base.WaitForShutdown() } From f2f9d2916b3a2f23761bbfab98940eb84c36f3b1 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 4 Apr 2022 17:26:01 +0100 Subject: [PATCH 46/55] Fix `dendritejs-pinecone` build --- cmd/dendritejs-pinecone/main.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/cmd/dendritejs-pinecone/main.go b/cmd/dendritejs-pinecone/main.go index 05cdf29ee..ba9edf230 100644 --- a/cmd/dendritejs-pinecone/main.go +++ b/cmd/dendritejs-pinecone/main.go @@ -21,8 +21,6 @@ import ( "crypto/ed25519" "encoding/hex" "fmt" - "log" - "os" "syscall/js" "time" @@ -154,9 +152,8 @@ func startup() { sk := generateKey() pk := sk.Public().(ed25519.PublicKey) - logger := log.New(os.Stdout, "", 0) - pRouter := pineconeRouter.NewRouter(logger, sk, false) - pSessions := pineconeSessions.NewSessions(logger, pRouter) + pRouter := pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk, false) + pSessions := pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), pRouter, []string{"matrix"}) cfg := &config.Dendrite{} cfg.Defaults(true) @@ -228,7 +225,7 @@ func startup() { httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux) httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux) - p2pRouter := pSessions.HTTP().Mux() + p2pRouter := pSessions.Protocol("matrix").HTTP().Mux() p2pRouter.Handle(httputil.PublicFederationPathPrefix, base.PublicFederationAPIMux) p2pRouter.Handle(httputil.PublicMediaPathPrefix, base.PublicMediaAPIMux) From 1783496423dcddd33fbbe4dcecdf8162c5cef685 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 4 Apr 2022 18:31:41 +0200 Subject: [PATCH 47/55] remove obsolete config entry "federation_certificates" (#2318) * Remove all federation_certificates occurencs This configuration value has not been used since 2019 apparently, and indeed it is never really used in the code base. So remove all traces of it from the various configuration files. Also remove the unused variable FederationCertificatePaths Signed-off-by: Sebastian Spaeth * setup/config/config_test.go: remove federation_sender config snippet The federation_sender: section was folded into the federation_api some time ago, and this seems to be the only leftover in the code base. So remove it. --- build/docker/config/dendrite.yaml | 6 ------ dendrite-config.yaml | 6 ------ internal/test/config.go | 2 -- setup/config/config_federationapi.go | 9 --------- setup/config/config_test.go | 12 ------------ 5 files changed, 35 deletions(-) diff --git a/build/docker/config/dendrite.yaml b/build/docker/config/dendrite.yaml index f3d373035..25cbd6d8c 100644 --- a/build/docker/config/dendrite.yaml +++ b/build/docker/config/dendrite.yaml @@ -173,12 +173,6 @@ federation_api: max_idle_conns: 2 conn_max_lifetime: -1 - # List of paths to X.509 certificates to be used by the external federation listeners. - # These certificates will be used to calculate the TLS fingerprints and other servers - # will expect the certificate to match these fingerprints. Certificates must be in PEM - # format. - federation_certificates: [] - # How many times we will try to resend a failed transaction to a specific server. The # backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds etc. send_max_retries: 16 diff --git a/dendrite-config.yaml b/dendrite-config.yaml index 6e2bc7be9..8b4c820a5 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -200,12 +200,6 @@ federation_api: max_idle_conns: 2 conn_max_lifetime: -1 - # List of paths to X.509 certificates to be used by the external federation listeners. - # These certificates will be used to calculate the TLS fingerprints and other servers - # will expect the certificate to match these fingerprints. Certificates must be in PEM - # format. - federation_certificates: [] - # How many times we will try to resend a failed transaction to a specific server. The # backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds etc. send_max_retries: 16 diff --git a/internal/test/config.go b/internal/test/config.go index 0b0e897b8..d8e0c4531 100644 --- a/internal/test/config.go +++ b/internal/test/config.go @@ -78,8 +78,6 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con cfg.Global.ServerName = gomatrixserverlib.ServerName(assignAddress()) cfg.Global.PrivateKeyPath = config.Path(serverKeyPath) - cfg.FederationAPI.FederationCertificatePaths = []config.Path{config.Path(tlsCertPath)} - cfg.MediaAPI.BasePath = config.Path(mediaBasePath) cfg.Global.JetStream.Addresses = []string{kafkaURI} diff --git a/setup/config/config_federationapi.go b/setup/config/config_federationapi.go index 95e705033..176334dd8 100644 --- a/setup/config/config_federationapi.go +++ b/setup/config/config_federationapi.go @@ -12,13 +12,6 @@ type FederationAPI struct { // send transactions to remote servers. Database DatabaseOptions `yaml:"database"` - // List of paths to X509 certificates used by the external federation listeners. - // These are used to calculate the TLS fingerprints to publish for this server. - // Other matrix servers talking to this server will expect the x509 certificate - // to match one of these certificates. - // The certificates should be in PEM format. - FederationCertificatePaths []Path `yaml:"federation_certificates"` - // Federation failure threshold. How many consecutive failures that we should // tolerate when sending federation requests to a specific server. The backoff // is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds, etc. @@ -57,8 +50,6 @@ func (c *FederationAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { checkURL(configErrs, "federation_api.external_api.listen", string(c.ExternalAPI.Listen)) } checkNotEmpty(configErrs, "federation_api.database.connection_string", string(c.Database.ConnectionString)) - // TODO: not applicable always, e.g. in demos - //checkNotZero(configErrs, "federation_api.federation_certificates", int64(len(c.FederationCertificatePaths))) } // The config for setting a proxy to use for server->server requests diff --git a/setup/config/config_test.go b/setup/config/config_test.go index 46e973fac..cbc57ad18 100644 --- a/setup/config/config_test.go +++ b/setup/config/config_test.go @@ -107,18 +107,6 @@ federation_api: connect: http://localhost:7772 external_api: listen: http://[::]:8072 - federation_certificates: [] -federation_sender: - internal_api: - listen: http://localhost:7775 - connect: http://localhost:7775 - database: - connection_string: file:federationapi.db - max_open_conns: 100 - max_idle_conns: 2 - conn_max_lifetime: -1 - send_max_retries: 16 - disable_tls_validation: false key_server: internal_api: listen: http://localhost:7779 From 562d7422405b180ef20555e61d0b3e4320d1577a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 4 Apr 2022 19:04:17 +0100 Subject: [PATCH 48/55] Update to matrix-org/gomatrixserverlib#299 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4830bf648..f287fb9f4 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20220317164600-0980b7f341e0 + github.com/matrix-org/gomatrixserverlib v0.0.0-20220404174134-970e11ad2142 github.com/matrix-org/pinecone v0.0.0-20220404141326-e526fa82f79d github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.10 diff --git a/go.sum b/go.sum index ae9f0fb7b..bbe109ff3 100644 --- a/go.sum +++ b/go.sum @@ -977,8 +977,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220317164600-0980b7f341e0 h1:IINbE/0jSYGb7M31StazufyIQdYWSivRlhuns3JYPOM= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220317164600-0980b7f341e0/go.mod h1:+WF5InseAMgi1fTnU46JH39IDpEvLep0fDzx9LDf2Bo= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220404174134-970e11ad2142 h1:kkFKjbPn9oySI07bA3vVInFMjTRdMxASgwJXmABli4o= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220404174134-970e11ad2142/go.mod h1:+WF5InseAMgi1fTnU46JH39IDpEvLep0fDzx9LDf2Bo= github.com/matrix-org/pinecone v0.0.0-20220404141326-e526fa82f79d h1:1+T4eOPRsf6cr0lMPW4oO2k8TTHm4mqIh65kpEID5Rk= github.com/matrix-org/pinecone v0.0.0-20220404141326-e526fa82f79d/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From 2defc4249d650f0bd43aa33b49b6b20032db43a4 Mon Sep 17 00:00:00 2001 From: David Spenler <15622190+DavidSpenler@users.noreply.github.com> Date: Tue, 5 Apr 2022 05:04:08 -0400 Subject: [PATCH 49/55] Added /upgrade endpoint (#2307) * Added /upgrade endpoint * fix * Fix lints * More lint lifex * Move room upgrading to the roomserver * Remove extraneous arg * Fix HTTP API for `PerformUpgrade` * Reduce number of API calls in `generateInitialEvents`, preserve membership fields * Refactor `generateInitialEvents` to preserve old state events for all but the essential room setup events * Handle ban events in the state transfer * Refactor and comment `createTemporaryPowerLevels` * Only send two power levels if we needed to override the levels, preserve miscellaneous fields in the create event * Fix copyrights * Review comments @S7evinK * Update sytest whitelist * Specify empty state keys, use `EventLevel`, remove unnecessary check on state copy * Add comment to `restrictOldRoomPowerLevels` * Ensure canonical aliases exist before clearing * Copy invites as well as bans * Fix return error on `m.room.tombstone` handling in client API * Relax checks for well-formedness of join rules, membership event etc Co-authored-by: Alex Kursell Co-authored-by: Neil Alexander Co-authored-by: kegsay --- clientapi/routing/routing.go | 10 + clientapi/routing/sendevent.go | 19 + clientapi/routing/upgrade_room.go | 92 +++ roomserver/api/api.go | 3 + roomserver/api/api_trace.go | 9 + roomserver/api/perform.go | 11 + roomserver/internal/api.go | 5 + .../internal/perform/perform_upgrade.go | 709 ++++++++++++++++++ roomserver/inthttp/client.go | 18 + roomserver/inthttp/server.go | 11 + sytest-whitelist | 18 + 11 files changed, 905 insertions(+) create mode 100644 clientapi/routing/upgrade_room.go create mode 100644 roomserver/internal/perform/perform_upgrade.go diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 8afaba560..db860dcdc 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -957,6 +957,16 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) + v3mux.Handle("/rooms/{roomID}/upgrade", + httputil.MakeAuthAPI("rooms_upgrade", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return UpgradeRoom(req, device, cfg, vars["roomID"], userAPI, rsAPI, asAPI) + }), + ).Methods(http.MethodPost, http.MethodOptions) + v3mux.Handle("/devices", httputil.MakeAuthAPI("get_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return GetDevicesByLocalpart(req, userAPI, device) diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index 3d5993718..c5884e80b 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -272,5 +272,24 @@ func generateSendEvent( JSON: jsonerror.Forbidden(err.Error()), // TODO: Is this error string comprehensible to the client? } } + + // User should not be able to send a tombstone event to the same room. + if e.Type() == "m.room.tombstone" { + content := make(map[string]interface{}) + if err = json.Unmarshal(e.Content(), &content); err != nil { + util.GetLogger(ctx).WithError(err).Error("Cannot unmarshal the event content.") + return nil, &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("Cannot unmarshal the event content."), + } + } + if content["replacement_room"] == e.RoomID() { + return nil, &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.InvalidParam("Cannot send tombstone event that points to the same room."), + } + } + } + return e.Event, nil } diff --git a/clientapi/routing/upgrade_room.go b/clientapi/routing/upgrade_room.go new file mode 100644 index 000000000..00bde36b3 --- /dev/null +++ b/clientapi/routing/upgrade_room.go @@ -0,0 +1,92 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "net/http" + + appserviceAPI "github.com/matrix-org/dendrite/appservice/api" + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/clientapi/jsonerror" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/version" + "github.com/matrix-org/dendrite/setup/config" + userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" +) + +type upgradeRoomRequest struct { + NewVersion string `json:"new_version"` +} + +type upgradeRoomResponse struct { + ReplacementRoom string `json:"replacement_room"` +} + +// UpgradeRoom implements /upgrade +func UpgradeRoom( + req *http.Request, device *userapi.Device, + cfg *config.ClientAPI, + roomID string, profileAPI userapi.UserProfileAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, + asAPI appserviceAPI.AppServiceQueryAPI, +) util.JSONResponse { + var r upgradeRoomRequest + if rErr := httputil.UnmarshalJSONRequest(req, &r); rErr != nil { + return *rErr + } + + // Validate that the room version is supported + if _, err := version.SupportedRoomVersion(gomatrixserverlib.RoomVersion(r.NewVersion)); err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.UnsupportedRoomVersion("This server does not support that room version"), + } + } + + upgradeReq := roomserverAPI.PerformRoomUpgradeRequest{ + UserID: device.UserID, + RoomID: roomID, + RoomVersion: gomatrixserverlib.RoomVersion(r.NewVersion), + } + upgradeResp := roomserverAPI.PerformRoomUpgradeResponse{} + + rsAPI.PerformRoomUpgrade(req.Context(), &upgradeReq, &upgradeResp) + + if upgradeResp.Error != nil { + if upgradeResp.Error.Code == roomserverAPI.PerformErrorNoRoom { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("Room does not exist"), + } + } else if upgradeResp.Error.Code == roomserverAPI.PerformErrorNotAllowed { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden(upgradeResp.Error.Msg), + } + } else { + return jsonerror.InternalServerError() + } + + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: upgradeRoomResponse{ + ReplacementRoom: upgradeResp.NewRoomID, + }, + } +} diff --git a/roomserver/api/api.go b/roomserver/api/api.go index bcbf0e4f9..fb77423f8 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -170,6 +170,9 @@ type RoomserverInternalAPI interface { // PerformForget forgets a rooms history for a specific user PerformForget(ctx context.Context, req *PerformForgetRequest, resp *PerformForgetResponse) error + // PerformRoomUpgrade upgrades a room to a newer version + PerformRoomUpgrade(ctx context.Context, req *PerformRoomUpgradeRequest, resp *PerformRoomUpgradeResponse) + // Asks for the default room version as preferred by the server. QueryRoomVersionCapabilities( ctx context.Context, diff --git a/roomserver/api/api_trace.go b/roomserver/api/api_trace.go index 88b372154..ec7211ef8 100644 --- a/roomserver/api/api_trace.go +++ b/roomserver/api/api_trace.go @@ -67,6 +67,15 @@ func (t *RoomserverInternalAPITrace) PerformUnpeek( util.GetLogger(ctx).Infof("PerformUnpeek req=%+v res=%+v", js(req), js(res)) } +func (t *RoomserverInternalAPITrace) PerformRoomUpgrade( + ctx context.Context, + req *PerformRoomUpgradeRequest, + res *PerformRoomUpgradeResponse, +) { + t.Impl.PerformRoomUpgrade(ctx, req, res) + util.GetLogger(ctx).Infof("PerformRoomUpgrade req=%+v res=%+v", js(req), js(res)) +} + func (t *RoomserverInternalAPITrace) PerformJoin( ctx context.Context, req *PerformJoinRequest, diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go index d640858a6..cda4b3ee4 100644 --- a/roomserver/api/perform.go +++ b/roomserver/api/perform.go @@ -203,3 +203,14 @@ type PerformForgetRequest struct { } type PerformForgetResponse struct{} + +type PerformRoomUpgradeRequest struct { + RoomID string `json:"room_id"` + UserID string `json:"user_id"` + RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` +} + +type PerformRoomUpgradeResponse struct { + NewRoomID string + Error *PerformError +} diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go index f96cefcb3..59f485cf7 100644 --- a/roomserver/internal/api.go +++ b/roomserver/internal/api.go @@ -34,6 +34,7 @@ type RoomserverInternalAPI struct { *perform.Publisher *perform.Backfiller *perform.Forgetter + *perform.Upgrader ProcessContext *process.ProcessContext DB storage.Database Cfg *config.RoomServer @@ -159,6 +160,10 @@ func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.FederationInternalA r.Forgetter = &perform.Forgetter{ DB: r.DB, } + r.Upgrader = &perform.Upgrader{ + Cfg: r.Cfg, + URSAPI: r, + } if err := r.Inputer.Start(); err != nil { logrus.WithError(err).Panic("failed to start roomserver input API") diff --git a/roomserver/internal/perform/perform_upgrade.go b/roomserver/internal/perform/perform_upgrade.go new file mode 100644 index 000000000..fcd19b936 --- /dev/null +++ b/roomserver/internal/perform/perform_upgrade.go @@ -0,0 +1,709 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package perform + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/matrix-org/dendrite/internal/eventutil" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/sirupsen/logrus" +) + +type Upgrader struct { + Cfg *config.RoomServer + URSAPI api.RoomserverInternalAPI +} + +// fledglingEvent is a helper representation of an event used when creating many events in succession. +type fledglingEvent struct { + Type string `json:"type"` + StateKey string `json:"state_key"` + Content interface{} `json:"content"` +} + +// PerformRoomUpgrade upgrades a room from one version to another +func (r *Upgrader) PerformRoomUpgrade( + ctx context.Context, + req *api.PerformRoomUpgradeRequest, + res *api.PerformRoomUpgradeResponse, +) { + res.NewRoomID, res.Error = r.performRoomUpgrade(ctx, req) + if res.Error != nil { + res.NewRoomID = "" + logrus.WithContext(ctx).WithError(res.Error).Error("Room upgrade failed") + } +} + +func (r *Upgrader) performRoomUpgrade( + ctx context.Context, + req *api.PerformRoomUpgradeRequest, +) (string, *api.PerformError) { + roomID := req.RoomID + userID := req.UserID + evTime := time.Now() + + // Return an immediate error if the room does not exist + if err := r.validateRoomExists(ctx, roomID); err != nil { + return "", &api.PerformError{ + Code: api.PerformErrorNoRoom, + Msg: "Error validating that the room exists", + } + } + + // 1. Check if the user is authorized to actually perform the upgrade (can send m.room.tombstone) + if !r.userIsAuthorized(ctx, userID, roomID) { + return "", &api.PerformError{ + Code: api.PerformErrorNotAllowed, + Msg: "You don't have permission to upgrade the room, power level too low.", + } + } + + // TODO (#267): Check room ID doesn't clash with an existing one, and we + // probably shouldn't be using pseudo-random strings, maybe GUIDs? + newRoomID := fmt.Sprintf("!%s:%s", util.RandomString(16), r.Cfg.Matrix.ServerName) + + // Get the existing room state for the old room. + oldRoomReq := &api.QueryLatestEventsAndStateRequest{ + RoomID: roomID, + } + oldRoomRes := &api.QueryLatestEventsAndStateResponse{} + if err := r.URSAPI.QueryLatestEventsAndState(ctx, oldRoomReq, oldRoomRes); err != nil { + return "", &api.PerformError{ + Msg: fmt.Sprintf("Failed to get latest state: %s", err), + } + } + + // Make the tombstone event + tombstoneEvent, pErr := r.makeTombstoneEvent(ctx, evTime, userID, roomID, newRoomID) + if pErr != nil { + return "", pErr + } + + // Generate the initial events we need to send into the new room. This includes copied state events and bans + // as well as the power level events needed to set up the room + eventsToMake, pErr := r.generateInitialEvents(ctx, oldRoomRes, userID, roomID, string(req.RoomVersion), tombstoneEvent) + if pErr != nil { + return "", pErr + } + + // 5. Send the tombstone event to the old room (must do this before we set the new canonical_alias) + if pErr = r.sendHeaderedEvent(ctx, tombstoneEvent); pErr != nil { + return "", pErr + } + + // Send the setup events to the new room + if pErr = r.sendInitialEvents(ctx, evTime, userID, newRoomID, string(req.RoomVersion), eventsToMake); pErr != nil { + return "", pErr + } + + // If the old room was public, make sure the new one is too + if pErr = r.publishIfOldRoomWasPublic(ctx, roomID, newRoomID); pErr != nil { + return "", pErr + } + + // If the old room had a canonical alias event, it should be deleted in the old room + if pErr = r.clearOldCanonicalAliasEvent(ctx, oldRoomRes, evTime, userID, roomID); pErr != nil { + return "", pErr + } + + // 4. Move local aliases to the new room + if pErr = moveLocalAliases(ctx, roomID, newRoomID, userID, r.URSAPI); pErr != nil { + return "", pErr + } + + // 6. Restrict power levels in the old room + if pErr = r.restrictOldRoomPowerLevels(ctx, evTime, userID, roomID); pErr != nil { + return "", pErr + } + + return newRoomID, nil +} + +func (r *Upgrader) getRoomPowerLevels(ctx context.Context, roomID string) (*gomatrixserverlib.PowerLevelContent, *api.PerformError) { + oldPowerLevelsEvent := api.GetStateEvent(ctx, r.URSAPI, roomID, gomatrixserverlib.StateKeyTuple{ + EventType: gomatrixserverlib.MRoomPowerLevels, + StateKey: "", + }) + powerLevelContent, err := oldPowerLevelsEvent.PowerLevels() + if err != nil { + util.GetLogger(ctx).WithError(err).Error() + return nil, &api.PerformError{ + Msg: "powerLevel event was not actually a power level event", + } + } + return powerLevelContent, nil +} + +func (r *Upgrader) restrictOldRoomPowerLevels(ctx context.Context, evTime time.Time, userID, roomID string) *api.PerformError { + restrictedPowerLevelContent, pErr := r.getRoomPowerLevels(ctx, roomID) + if pErr != nil { + return pErr + } + + // From: https://spec.matrix.org/v1.2/client-server-api/#server-behaviour-16 + // If possible, the power levels in the old room should also be modified to + // prevent sending of events and inviting new users. For example, setting + // events_default and invite to the greater of 50 and users_default + 1. + restrictedDefaultPowerLevel := int64(50) + if restrictedPowerLevelContent.UsersDefault+1 > restrictedDefaultPowerLevel { + restrictedDefaultPowerLevel = restrictedPowerLevelContent.UsersDefault + 1 + } + restrictedPowerLevelContent.EventsDefault = restrictedDefaultPowerLevel + restrictedPowerLevelContent.Invite = restrictedDefaultPowerLevel + + restrictedPowerLevelsHeadered, resErr := r.makeHeaderedEvent(ctx, evTime, userID, roomID, fledglingEvent{ + Type: gomatrixserverlib.MRoomPowerLevels, + StateKey: "", + Content: restrictedPowerLevelContent, + }) + if resErr != nil { + if resErr.Code == api.PerformErrorNotAllowed { + util.GetLogger(ctx).WithField(logrus.ErrorKey, resErr).Warn("UpgradeRoom: Could not restrict power levels in old room") + } else { + return resErr + } + } else { + if resErr = r.sendHeaderedEvent(ctx, restrictedPowerLevelsHeadered); resErr != nil { + return resErr + } + } + return nil +} + +func moveLocalAliases(ctx context.Context, + roomID, newRoomID, userID string, + URSAPI api.RoomserverInternalAPI) *api.PerformError { + var err error + + aliasReq := api.GetAliasesForRoomIDRequest{RoomID: roomID} + aliasRes := api.GetAliasesForRoomIDResponse{} + if err = URSAPI.GetAliasesForRoomID(ctx, &aliasReq, &aliasRes); err != nil { + return &api.PerformError{ + Msg: "Could not get aliases for old room", + } + } + + for _, alias := range aliasRes.Aliases { + removeAliasReq := api.RemoveRoomAliasRequest{UserID: userID, Alias: alias} + removeAliasRes := api.RemoveRoomAliasResponse{} + if err = URSAPI.RemoveRoomAlias(ctx, &removeAliasReq, &removeAliasRes); err != nil { + return &api.PerformError{ + Msg: "api.RemoveRoomAlias failed", + } + } + + setAliasReq := api.SetRoomAliasRequest{UserID: userID, Alias: alias, RoomID: newRoomID} + setAliasRes := api.SetRoomAliasResponse{} + if err = URSAPI.SetRoomAlias(ctx, &setAliasReq, &setAliasRes); err != nil { + return &api.PerformError{ + Msg: "api.SetRoomAlias failed", + } + } + } + return nil +} + +func (r *Upgrader) clearOldCanonicalAliasEvent(ctx context.Context, oldRoom *api.QueryLatestEventsAndStateResponse, evTime time.Time, userID, roomID string) *api.PerformError { + for _, event := range oldRoom.StateEvents { + if event.Type() != gomatrixserverlib.MRoomCanonicalAlias || !event.StateKeyEquals("") { + continue + } + var aliasContent struct { + Alias string `json:"alias"` + AltAliases []string `json:"alt_aliases"` + } + if err := json.Unmarshal(event.Content(), &aliasContent); err != nil { + return &api.PerformError{ + Msg: fmt.Sprintf("Failed to unmarshal canonical aliases: %s", err), + } + } + if aliasContent.Alias == "" && len(aliasContent.AltAliases) == 0 { + // There are no canonical aliases to clear, therefore do nothing. + return nil + } + } + + emptyCanonicalAliasEvent, resErr := r.makeHeaderedEvent(ctx, evTime, userID, roomID, fledglingEvent{ + Type: gomatrixserverlib.MRoomCanonicalAlias, + Content: map[string]interface{}{}, + }) + if resErr != nil { + if resErr.Code == api.PerformErrorNotAllowed { + util.GetLogger(ctx).WithField(logrus.ErrorKey, resErr).Warn("UpgradeRoom: Could not set empty canonical alias event in old room") + } else { + return resErr + } + } else { + if resErr = r.sendHeaderedEvent(ctx, emptyCanonicalAliasEvent); resErr != nil { + return resErr + } + } + return nil +} + +func (r *Upgrader) publishIfOldRoomWasPublic(ctx context.Context, roomID, newRoomID string) *api.PerformError { + // check if the old room was published + var pubQueryRes api.QueryPublishedRoomsResponse + err := r.URSAPI.QueryPublishedRooms(ctx, &api.QueryPublishedRoomsRequest{ + RoomID: roomID, + }, &pubQueryRes) + if err != nil { + return &api.PerformError{ + Msg: "QueryPublishedRooms failed", + } + } + + // if the old room is published (was public), publish the new room + if len(pubQueryRes.RoomIDs) == 1 { + publishNewRoomAndUnpublishOldRoom(ctx, r.URSAPI, roomID, newRoomID) + } + return nil +} + +func publishNewRoomAndUnpublishOldRoom( + ctx context.Context, + URSAPI api.RoomserverInternalAPI, + oldRoomID, newRoomID string, +) { + // expose this room in the published room list + var pubNewRoomRes api.PerformPublishResponse + URSAPI.PerformPublish(ctx, &api.PerformPublishRequest{ + RoomID: newRoomID, + Visibility: "public", + }, &pubNewRoomRes) + if pubNewRoomRes.Error != nil { + // treat as non-fatal since the room is already made by this point + util.GetLogger(ctx).WithError(pubNewRoomRes.Error).Error("failed to visibility:public") + } + + var unpubOldRoomRes api.PerformPublishResponse + // remove the old room from the published room list + URSAPI.PerformPublish(ctx, &api.PerformPublishRequest{ + RoomID: oldRoomID, + Visibility: "private", + }, &unpubOldRoomRes) + if unpubOldRoomRes.Error != nil { + // treat as non-fatal since the room is already made by this point + util.GetLogger(ctx).WithError(unpubOldRoomRes.Error).Error("failed to visibility:private") + } +} + +func (r *Upgrader) validateRoomExists(ctx context.Context, roomID string) error { + verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} + verRes := api.QueryRoomVersionForRoomResponse{} + if err := r.URSAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil { + return &api.PerformError{ + Code: api.PerformErrorNoRoom, + Msg: "Room does not exist", + } + } + return nil +} + +func (r *Upgrader) userIsAuthorized(ctx context.Context, userID, roomID string, +) bool { + plEvent := api.GetStateEvent(ctx, r.URSAPI, roomID, gomatrixserverlib.StateKeyTuple{ + EventType: gomatrixserverlib.MRoomPowerLevels, + StateKey: "", + }) + if plEvent == nil { + return false + } + pl, err := plEvent.PowerLevels() + if err != nil { + return false + } + // Check for power level required to send tombstone event (marks the current room as obsolete), + // if not found, use the StateDefault power level + return pl.UserLevel(userID) >= pl.EventLevel("m.room.tombstone", true) +} + +// nolint:gocyclo +func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.QueryLatestEventsAndStateResponse, userID, roomID, newVersion string, tombstoneEvent *gomatrixserverlib.HeaderedEvent) ([]fledglingEvent, *api.PerformError) { + state := make(map[gomatrixserverlib.StateKeyTuple]*gomatrixserverlib.HeaderedEvent, len(oldRoom.StateEvents)) + for _, event := range oldRoom.StateEvents { + if event.StateKey() == nil { + // This shouldn't ever happen, but better to be safe than sorry. + continue + } + if event.Type() == gomatrixserverlib.MRoomMember && !event.StateKeyEquals(userID) { + // With the exception of bans and invites which we do want to copy, we + // should ignore membership events that aren't our own, as event auth will + // prevent us from being able to create membership events on behalf of other + // users anyway unless they are invites or bans. + membership, err := event.Membership() + if err != nil { + continue + } + switch membership { + case gomatrixserverlib.Ban: + case gomatrixserverlib.Invite: + default: + continue + } + } + state[gomatrixserverlib.StateKeyTuple{EventType: event.Type(), StateKey: *event.StateKey()}] = event + } + + // The following events are ones that we are going to override manually + // in the following section. + override := map[gomatrixserverlib.StateKeyTuple]struct{}{ + {EventType: gomatrixserverlib.MRoomCreate, StateKey: ""}: {}, + {EventType: gomatrixserverlib.MRoomMember, StateKey: userID}: {}, + {EventType: gomatrixserverlib.MRoomPowerLevels, StateKey: ""}: {}, + {EventType: gomatrixserverlib.MRoomJoinRules, StateKey: ""}: {}, + } + + // The overridden events are essential events that must be present in the + // old room state. Check that they are there. + for tuple := range override { + if _, ok := state[tuple]; !ok { + return nil, &api.PerformError{ + Msg: fmt.Sprintf("Essential event of type %q state key %q is missing", tuple.EventType, tuple.StateKey), + } + } + } + + oldCreateEvent := state[gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomCreate, StateKey: ""}] + oldMembershipEvent := state[gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomMember, StateKey: userID}] + oldPowerLevelsEvent := state[gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomPowerLevels, StateKey: ""}] + oldJoinRulesEvent := state[gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomJoinRules, StateKey: ""}] + + // Create the new room create event. Using a map here instead of CreateContent + // means that we preserve any other interesting fields that might be present + // in the create event (such as for the room types MSC). + newCreateContent := map[string]interface{}{} + _ = json.Unmarshal(oldCreateEvent.Content(), &newCreateContent) + newCreateContent["creator"] = userID + newCreateContent["room_version"] = newVersion + newCreateContent["predecessor"] = gomatrixserverlib.PreviousRoom{ + EventID: tombstoneEvent.EventID(), + RoomID: roomID, + } + newCreateEvent := fledglingEvent{ + Type: gomatrixserverlib.MRoomCreate, + StateKey: "", + Content: newCreateContent, + } + + // Now create the new membership event. Same rules apply as above, so + // that we preserve fields we don't otherwise know about. We'll always + // set the membership to join though, because that is necessary to auth + // the events after it. + newMembershipContent := map[string]interface{}{} + _ = json.Unmarshal(oldMembershipEvent.Content(), &newMembershipContent) + newMembershipContent["membership"] = gomatrixserverlib.Join + newMembershipEvent := fledglingEvent{ + Type: gomatrixserverlib.MRoomMember, + StateKey: userID, + Content: newMembershipContent, + } + + // We might need to temporarily give ourselves a higher power level + // than we had in the old room in order to be able to send all of + // the relevant state events. This function will return whether we + // had to override the power level events or not — if we did, we + // need to send the original power levels again later on. + powerLevelContent, err := oldPowerLevelsEvent.PowerLevels() + if err != nil { + util.GetLogger(ctx).WithError(err).Error() + return nil, &api.PerformError{ + Msg: "Power level event content was invalid", + } + } + tempPowerLevelsEvent, powerLevelsOverridden := createTemporaryPowerLevels(powerLevelContent, userID) + + // Now do the join rules event, same as the create and membership + // events. We'll set a sane default of "invite" so that if the + // existing join rules contains garbage, the room can still be + // upgraded. + newJoinRulesContent := map[string]interface{}{ + "join_rule": gomatrixserverlib.Invite, // sane default + } + _ = json.Unmarshal(oldJoinRulesEvent.Content(), &newJoinRulesContent) + newJoinRulesEvent := fledglingEvent{ + Type: gomatrixserverlib.MRoomJoinRules, + StateKey: "", + Content: newJoinRulesContent, + } + + eventsToMake := make([]fledglingEvent, 0, len(state)) + eventsToMake = append( + eventsToMake, newCreateEvent, newMembershipEvent, + tempPowerLevelsEvent, newJoinRulesEvent, + ) + + // For some reason Sytest expects there to be a guest access event. + // Create one if it doesn't exist. + if _, ok := state[gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomGuestAccess, StateKey: ""}]; !ok { + eventsToMake = append(eventsToMake, fledglingEvent{ + Type: gomatrixserverlib.MRoomGuestAccess, + Content: map[string]string{ + "guest_access": "forbidden", + }, + }) + } + + // Duplicate all of the old state events into the new room. + for tuple, event := range state { + if _, ok := override[tuple]; ok { + // Don't duplicate events we have overridden already. They + // are already in `eventsToMake`. + continue + } + newEvent := fledglingEvent{ + Type: tuple.EventType, + StateKey: tuple.StateKey, + } + if err = json.Unmarshal(event.Content(), &newEvent.Content); err != nil { + logrus.WithError(err).Error("Failed to unmarshal old event") + continue + } + eventsToMake = append(eventsToMake, newEvent) + } + + // If we sent a temporary power level event into the room before, + // override that now by restoring the original power levels. + if powerLevelsOverridden { + eventsToMake = append(eventsToMake, fledglingEvent{ + Type: gomatrixserverlib.MRoomPowerLevels, + Content: powerLevelContent, + }) + } + return eventsToMake, nil +} + +func (r *Upgrader) sendInitialEvents(ctx context.Context, evTime time.Time, userID, newRoomID, newVersion string, eventsToMake []fledglingEvent) *api.PerformError { + var err error + var builtEvents []*gomatrixserverlib.HeaderedEvent + authEvents := gomatrixserverlib.NewAuthEvents(nil) + for i, e := range eventsToMake { + depth := i + 1 // depth starts at 1 + + builder := gomatrixserverlib.EventBuilder{ + Sender: userID, + RoomID: newRoomID, + Type: e.Type, + StateKey: &e.StateKey, + Depth: int64(depth), + } + err = builder.SetContent(e.Content) + if err != nil { + return &api.PerformError{ + Msg: "builder.SetContent failed", + } + } + if i > 0 { + builder.PrevEvents = []gomatrixserverlib.EventReference{builtEvents[i-1].EventReference()} + } + var event *gomatrixserverlib.Event + event, err = r.buildEvent(&builder, &authEvents, evTime, gomatrixserverlib.RoomVersion(newVersion)) + if err != nil { + return &api.PerformError{ + Msg: "buildEvent failed", + } + } + + if err = gomatrixserverlib.Allowed(event, &authEvents); err != nil { + return &api.PerformError{ + Msg: "gomatrixserverlib.Allowed failed", + } + } + + // Add the event to the list of auth events + builtEvents = append(builtEvents, event.Headered(gomatrixserverlib.RoomVersion(newVersion))) + err = authEvents.AddEvent(event) + if err != nil { + return &api.PerformError{ + Msg: "authEvents.AddEvent failed", + } + } + } + + inputs := make([]api.InputRoomEvent, 0, len(builtEvents)) + for _, event := range builtEvents { + inputs = append(inputs, api.InputRoomEvent{ + Kind: api.KindNew, + Event: event, + Origin: r.Cfg.Matrix.ServerName, + SendAsServer: api.DoNotSendToOtherServers, + }) + } + if err = api.SendInputRoomEvents(ctx, r.URSAPI, inputs, false); err != nil { + return &api.PerformError{ + Msg: "api.SendInputRoomEvents failed", + } + } + return nil +} + +func (r *Upgrader) makeTombstoneEvent( + ctx context.Context, + evTime time.Time, + userID, roomID, newRoomID string, +) (*gomatrixserverlib.HeaderedEvent, *api.PerformError) { + content := map[string]interface{}{ + "body": "This room has been replaced", + "replacement_room": newRoomID, + } + event := fledglingEvent{ + Type: "m.room.tombstone", + Content: content, + } + return r.makeHeaderedEvent(ctx, evTime, userID, roomID, event) +} + +func (r *Upgrader) makeHeaderedEvent(ctx context.Context, evTime time.Time, userID, roomID string, event fledglingEvent) (*gomatrixserverlib.HeaderedEvent, *api.PerformError) { + builder := gomatrixserverlib.EventBuilder{ + Sender: userID, + RoomID: roomID, + Type: event.Type, + StateKey: &event.StateKey, + } + err := builder.SetContent(event.Content) + if err != nil { + return nil, &api.PerformError{ + Msg: "builder.SetContent failed", + } + } + var queryRes api.QueryLatestEventsAndStateResponse + headeredEvent, err := eventutil.QueryAndBuildEvent(ctx, &builder, r.Cfg.Matrix, evTime, r.URSAPI, &queryRes) + if err == eventutil.ErrRoomNoExists { + return nil, &api.PerformError{ + Code: api.PerformErrorNoRoom, + Msg: "Room does not exist", + } + } else if e, ok := err.(gomatrixserverlib.BadJSONError); ok { + return nil, &api.PerformError{ + Msg: e.Error(), + } + } else if e, ok := err.(gomatrixserverlib.EventValidationError); ok { + if e.Code == gomatrixserverlib.EventValidationTooLarge { + return nil, &api.PerformError{ + Msg: e.Error(), + } + } + return nil, &api.PerformError{ + Msg: e.Error(), + } + } else if err != nil { + return nil, &api.PerformError{ + Msg: "eventutil.BuildEvent failed", + } + } + // check to see if this user can perform this operation + stateEvents := make([]*gomatrixserverlib.Event, len(queryRes.StateEvents)) + for i := range queryRes.StateEvents { + stateEvents[i] = queryRes.StateEvents[i].Event + } + provider := gomatrixserverlib.NewAuthEvents(stateEvents) + if err = gomatrixserverlib.Allowed(headeredEvent.Event, &provider); err != nil { + return nil, &api.PerformError{ + Code: api.PerformErrorNotAllowed, + Msg: err.Error(), // TODO: Is this error string comprehensible to the client? + } + } + + return headeredEvent, nil +} + +func createTemporaryPowerLevels(powerLevelContent *gomatrixserverlib.PowerLevelContent, userID string) (fledglingEvent, bool) { + // Work out what power level we need in order to be able to send events + // of all types into the room. + neededPowerLevel := powerLevelContent.StateDefault + for _, powerLevel := range powerLevelContent.Events { + if powerLevel > neededPowerLevel { + neededPowerLevel = powerLevel + } + } + + // Make a copy of the existing power level content. + tempPowerLevelContent := *powerLevelContent + powerLevelsOverridden := false + + // At this point, the "Users", "Events" and "Notifications" keys are all + // pointing to the map of the original PL content, so we will specifically + // override the users map with a new one and duplicate the values deeply, + // so that we can modify them without modifying the original. + tempPowerLevelContent.Users = make(map[string]int64, len(powerLevelContent.Users)) + for key, value := range powerLevelContent.Users { + tempPowerLevelContent.Users[key] = value + } + + // If the user who is upgrading the room doesn't already have sufficient + // power, then elevate their power levels. + if tempPowerLevelContent.UserLevel(userID) < neededPowerLevel { + tempPowerLevelContent.Users[userID] = neededPowerLevel + powerLevelsOverridden = true + } + + // Then return the temporary power levels event. + return fledglingEvent{ + Type: gomatrixserverlib.MRoomPowerLevels, + Content: tempPowerLevelContent, + }, powerLevelsOverridden +} + +func (r *Upgrader) sendHeaderedEvent( + ctx context.Context, + headeredEvent *gomatrixserverlib.HeaderedEvent, +) *api.PerformError { + var inputs []api.InputRoomEvent + inputs = append(inputs, api.InputRoomEvent{ + Kind: api.KindNew, + Event: headeredEvent, + Origin: r.Cfg.Matrix.ServerName, + SendAsServer: api.DoNotSendToOtherServers, + }) + if err := api.SendInputRoomEvents(ctx, r.URSAPI, inputs, false); err != nil { + return &api.PerformError{ + Msg: "api.SendInputRoomEvents failed", + } + } + + return nil +} + +func (r *Upgrader) buildEvent( + builder *gomatrixserverlib.EventBuilder, + provider gomatrixserverlib.AuthEventProvider, + evTime time.Time, + roomVersion gomatrixserverlib.RoomVersion, +) (*gomatrixserverlib.Event, error) { + eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder) + if err != nil { + return nil, err + } + refs, err := eventsNeeded.AuthEventReferences(provider) + if err != nil { + return nil, err + } + builder.AuthEvents = refs + event, err := builder.Build( + evTime, r.Cfg.Matrix.ServerName, r.Cfg.Matrix.KeyID, + r.Cfg.Matrix.PrivateKey, roomVersion, + ) + if err != nil { + return nil, fmt.Errorf("cannot build event %s : Builder failed to build. %w", builder.Type, err) + } + return event, nil +} diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go index 99c596606..d55805a91 100644 --- a/roomserver/inthttp/client.go +++ b/roomserver/inthttp/client.go @@ -32,6 +32,7 @@ const ( RoomserverPerformInvitePath = "/roomserver/performInvite" RoomserverPerformPeekPath = "/roomserver/performPeek" RoomserverPerformUnpeekPath = "/roomserver/performUnpeek" + RoomserverPerformRoomUpgradePath = "/roomserver/performRoomUpgrade" RoomserverPerformJoinPath = "/roomserver/performJoin" RoomserverPerformLeavePath = "/roomserver/performLeave" RoomserverPerformBackfillPath = "/roomserver/performBackfill" @@ -252,6 +253,23 @@ func (h *httpRoomserverInternalAPI) PerformUnpeek( } } +func (h *httpRoomserverInternalAPI) PerformRoomUpgrade( + ctx context.Context, + request *api.PerformRoomUpgradeRequest, + response *api.PerformRoomUpgradeResponse, +) { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformRoomUpgrade") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverPerformRoomUpgradePath + err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + if err != nil { + response.Error = &api.PerformError{ + Msg: fmt.Sprintf("failed to communicate with roomserver: %s", err), + } + } +} + func (h *httpRoomserverInternalAPI) PerformLeave( ctx context.Context, request *api.PerformLeaveRequest, diff --git a/roomserver/inthttp/server.go b/roomserver/inthttp/server.go index 691a45830..0b27b5a8d 100644 --- a/roomserver/inthttp/server.go +++ b/roomserver/inthttp/server.go @@ -96,6 +96,17 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + internalAPIMux.Handle(RoomserverPerformRoomUpgradePath, + httputil.MakeInternalAPI("performRoomUpgrade", func(req *http.Request) util.JSONResponse { + var request api.PerformRoomUpgradeRequest + var response api.PerformRoomUpgradeResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + r.PerformRoomUpgrade(req.Context(), &request, &response) + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) internalAPIMux.Handle(RoomserverPerformPublishPath, httputil.MakeInternalAPI("performPublish", func(req *http.Request) util.JSONResponse { var request api.PerformPublishRequest diff --git a/sytest-whitelist b/sytest-whitelist index 7614fbb96..5baac2ad4 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -661,4 +661,22 @@ Canonical alias can include alt_aliases Can delete canonical alias AS can make room aliases /context/ with lazy_load_members filter works +/upgrade creates a new room +/upgrade should preserve room visibility for public rooms +/upgrade should preserve room visibility for private rooms +/upgrade copies the power levels to the new room +/upgrade preserves the power level of the upgrading user in old and new rooms +/upgrade copies important state to the new room +/upgrade copies ban events to the new room +local user has push rules copied to upgraded room +remote user has push rules copied to upgraded room +/upgrade moves aliases to the new room +/upgrade preserves room federation ability +/upgrade restricts power levels in the old room +/upgrade restricts power levels in the old room when the old PLs are unusual +/upgrade to an unknown version is rejected +/upgrade is rejected if the user can't send state events +/upgrade of a bogus room fails gracefully +Cannot send tombstone event that points to the same room Room summary counts change when membership changes +/upgrade copies >100 power levels to the new room From fb10633a80395b56ad92f0a82adcfbe2ed809cb3 Mon Sep 17 00:00:00 2001 From: Emanuele Aliberti Date: Tue, 5 Apr 2022 12:27:29 +0200 Subject: [PATCH 50/55] build.sh to build.cmd (#2319) * build.sh to build.cmd convert and adapt for Windows users * remove hardwired GO ARCH * silence cmd.exe * update directions for Windows * Update INSTALL.md Co-authored-by: emanuele.aliberti Co-authored-by: Neil Alexander --- build.cmd | 51 +++++++++++++++++++++++++++++++++++++++++++++++++ docs/INSTALL.md | 16 +++++++++++++--- 2 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 build.cmd diff --git a/build.cmd b/build.cmd new file mode 100644 index 000000000..9e90622c8 --- /dev/null +++ b/build.cmd @@ -0,0 +1,51 @@ +@echo off + +:ENTRY_POINT + setlocal EnableDelayedExpansion + + REM script base dir + set SCRIPTDIR=%~dp0 + set PROJDIR=%SCRIPTDIR:~0,-1% + + REM Put installed packages into ./bin + set GOBIN=%PROJDIR%\bin + + set FLAGS= + + REM Check if sources are under Git control + if not exist ".git" goto :CHECK_BIN + + REM set BUILD=`git rev-parse --short HEAD \\ ""` + FOR /F "tokens=*" %%X IN ('git rev-parse --short HEAD') DO ( + set BUILD=%%X + ) + + REM set BRANCH=`(git symbolic-ref --short HEAD \ tr -d \/ ) \\ ""` + FOR /F "tokens=*" %%X IN ('git symbolic-ref --short HEAD') DO ( + set BRANCHRAW=%%X + set BRANCH=!BRANCHRAW:/=! + ) + if "%BRANCH%" == "main" set BRANCH= + + set FLAGS=-X github.com/matrix-org/dendrite/internal.branch=%BRANCH% -X github.com/matrix-org/dendrite/internal.build=%BUILD% + +:CHECK_BIN + if exist "bin" goto :ALL_SET + mkdir "bin" + +:ALL_SET + set CGO_ENABLED=1 + for /D %%P in (cmd\*) do ( + go build -trimpath -ldflags "%FLAGS%" -v -o ".\bin" ".\%%P" + ) + + set CGO_ENABLED=0 + set GOOS=js + set GOARCH=wasm + go build -trimpath -ldflags "%FLAGS%" -o bin\main.wasm .\cmd\dendritejs-pinecone + + goto :DONE + +:DONE + echo Done + endlocal \ No newline at end of file diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 523c5c7d1..ca1316aca 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -34,6 +34,10 @@ If you want to run a polylith deployment, you also need: * A standalone [NATS Server](https://github.com/nats-io/nats-server) deployment with JetStream enabled +If you want to build it on Windows, you need `gcc` in the path: + +* [MinGW-w64](https://www.mingw-w64.org/) + ## Building Dendrite Start by cloning the code: @@ -45,9 +49,15 @@ cd dendrite Then build it: -```bash -./build.sh -``` +* Linux or UNIX-like systems: + ```bash + ./build.sh + ``` + +* Windows: + ```dos + build.cmd + ``` ## Install NATS Server From 47be39c18e4c76f93fc35978b534ef178aafdbcf Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 5 Apr 2022 11:49:29 +0100 Subject: [PATCH 51/55] Update sytest-whitelist again --- sytest-whitelist | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sytest-whitelist b/sytest-whitelist index 5baac2ad4..38a057da6 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -680,3 +680,5 @@ remote user has push rules copied to upgraded room Cannot send tombstone event that points to the same room Room summary counts change when membership changes /upgrade copies >100 power levels to the new room +Room state after a rejected message event is the same as before +Room state after a rejected state event is the same as before \ No newline at end of file From c84937b85292c31411eb2329570fdbf1903582e7 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Tue, 5 Apr 2022 15:32:30 +0200 Subject: [PATCH 52/55] Add Are We Synapse Yet to GHA (#2321) * Add Are We Synapse Yet to GHA * Better output & add comments --- .github/workflows/dendrite.yml | 9 ++++++++- are-we-synapse-yet.py | 9 +++++---- show-expected-fail-tests.sh | 16 ++++++++-------- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index 99747f798..c80a82d1d 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -233,7 +233,14 @@ jobs: - name: Summarise results.tap if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - + - name: Sytest List Maintenance + if: ${{ always() }} + run: /src/show-expected-fail-tests.sh /logs/results.tap /src/sytest-whitelist /src/sytest-blacklist + continue-on-error: true # not fatal + - name: Are We Synapse Yet? + if: ${{ always() }} + run: /src/are-we-synapse-yet.py /logs/results.tap -v + continue-on-error: true # not fatal - name: Upload Sytest logs uses: actions/upload-artifact@v2 if: ${{ always() }} diff --git a/are-we-synapse-yet.py b/are-we-synapse-yet.py index 10b1be28a..8d551575f 100755 --- a/are-we-synapse-yet.py +++ b/are-we-synapse-yet.py @@ -3,7 +3,7 @@ from __future__ import division import argparse import re -import sys +import os # Usage: $ ./are-we-synapse-yet.py [-v] results.tap # This script scans a results.tap file from Dendrite's CI process and spits out @@ -156,6 +156,7 @@ def parse_test_line(line): # ✓ POST /register downcases capitals in usernames # ... def print_stats(header_name, gid_to_tests, gid_to_name, verbose): + ci = os.getenv("CI") # When running from GHA, this groups the subsections subsections = [] # Registration: 100% (13/13 tests) subsection_test_names = {} # 'subsection name': ["✓ Test 1", "✓ Test 2", "× Test 3"] total_passing = 0 @@ -169,7 +170,7 @@ def print_stats(header_name, gid_to_tests, gid_to_name, verbose): for name, passing in tests.items(): if passing: group_passing += 1 - test_names_and_marks.append(f"{'✓' if passing else '×'} {name}") + test_names_and_marks.append(f"{'✅' if passing else '❌'} {name}") total_tests += group_total total_passing += group_passing @@ -186,11 +187,11 @@ def print_stats(header_name, gid_to_tests, gid_to_name, verbose): print("%s: %s (%d/%d tests)" % (header_name, pct, total_passing, total_tests)) print("-" * (len(header_name)+1)) for line in subsections: - print(" %s" % (line,)) + print("%s%s" % ("::group::" if ci and verbose else "", line,)) if verbose: for test_name_and_pass_mark in subsection_test_names[line]: print(" %s" % (test_name_and_pass_mark,)) - print("") + print("%s" % ("::endgroup::" if ci else "")) print("") def main(results_tap_path, verbose): diff --git a/show-expected-fail-tests.sh b/show-expected-fail-tests.sh index 320d4ebd3..3ed937a0f 100755 --- a/show-expected-fail-tests.sh +++ b/show-expected-fail-tests.sh @@ -89,17 +89,17 @@ if [ -n "${tests_to_add}" ] && [ -n "${already_in_whitelist}" ]; then fi if [ -n "${tests_to_add}" ]; then - echo "**ERROR**: The following tests passed but are not present in \`$2\`. Please append them to the file:" - echo "\`\`\`" - echo -e "${tests_to_add}" - echo "\`\`\`" + echo "::error::The following tests passed but are not present in \`$2\`. Please append them to the file:" + echo "::group::Passing tests" + echo -e "${tests_to_add}" + echo "::endgroup::" fi if [ -n "${already_in_whitelist}" ]; then - echo "**WARN**: Tests in the whitelist still marked as **expected fail**:" - echo "\`\`\`" - echo -e "${already_in_whitelist}" - echo "\`\`\`" + echo "::warning::Tests in the whitelist still marked as **expected fail**:" + echo "::group::Still marked as expected fail" + echo -e "${already_in_whitelist}" + echo "::endgroup::" fi exit ${fail_build} From 4d9d9cc9b105ec5a4cd64dbaca000d0592c8c226 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 5 Apr 2022 14:43:44 +0100 Subject: [PATCH 53/55] Update to matrix-org/gomatrixserverlib#300 --- federationapi/internal/perform.go | 10 +++++----- go.mod | 3 +-- go.sum | 8 ++++---- roomserver/internal/input/input_missing.go | 7 ++++--- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go index b888b3654..8cd944346 100644 --- a/federationapi/internal/perform.go +++ b/federationapi/internal/perform.go @@ -392,17 +392,17 @@ func (r *FederationInternalAPI) performOutboundPeekUsingServer( // we have the peek state now so let's process regardless of whether upstream gives up ctx = context.Background() - respState := respPeek.ToRespState() - authEvents := respState.AuthEvents.UntrustedEvents(respPeek.RoomVersion) + // authenticate the state returned (check its auth events etc) // the equivalent of CheckSendJoinResponse() + authEvents, _, err := respState.Check(ctx, respPeek.RoomVersion, r.keyRing, federatedAuthProvider(ctx, r.federation, r.keyRing, serverName)) + if err != nil { + return fmt.Errorf("error checking state returned from peeking: %w", err) + } if err = sanityCheckAuthChain(authEvents); err != nil { return fmt.Errorf("sanityCheckAuthChain: %w", err) } - if err = respState.Check(ctx, respPeek.RoomVersion, r.keyRing, federatedAuthProvider(ctx, r.federation, r.keyRing, serverName)); err != nil { - return fmt.Errorf("error checking state returned from peeking: %w", err) - } // If we've got this far, the remote server is peeking. if renewing { diff --git a/go.mod b/go.mod index f287fb9f4..a34f8a578 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20220404174134-970e11ad2142 + github.com/matrix-org/gomatrixserverlib v0.0.0-20220405134050-301e340659d5 github.com/matrix-org/pinecone v0.0.0-20220404141326-e526fa82f79d github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.10 @@ -64,7 +64,6 @@ require ( golang.org/x/image v0.0.0-20211028202545-6944b10bf410 golang.org/x/mobile v0.0.0-20220325161704-447654d348e3 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd - golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 gopkg.in/h2non/bimg.v1 v1.1.5 gopkg.in/yaml.v2 v2.4.0 diff --git a/go.sum b/go.sum index bbe109ff3..c76f9c55d 100644 --- a/go.sum +++ b/go.sum @@ -977,8 +977,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220404174134-970e11ad2142 h1:kkFKjbPn9oySI07bA3vVInFMjTRdMxASgwJXmABli4o= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220404174134-970e11ad2142/go.mod h1:+WF5InseAMgi1fTnU46JH39IDpEvLep0fDzx9LDf2Bo= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220405134050-301e340659d5 h1:Fkennny7+Z/5pygrhjFMZbz1j++P2hhhLoT7NO3p8DQ= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220405134050-301e340659d5/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48= github.com/matrix-org/pinecone v0.0.0-20220404141326-e526fa82f79d h1:1+T4eOPRsf6cr0lMPW4oO2k8TTHm4mqIh65kpEID5Rk= github.com/matrix-org/pinecone v0.0.0-20220404141326-e526fa82f79d/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= @@ -1727,8 +1727,8 @@ golang.org/x/sys v0.0.0-20211102192858-4dd72447c267/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64 h1:D1v9ucDTYBtbz5vNuBbAhIMAGhQhJ6Ym5ah3maMVNX4= +golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/roomserver/internal/input/input_missing.go b/roomserver/internal/input/input_missing.go index a7da9b06d..2c958335d 100644 --- a/roomserver/internal/input/input_missing.go +++ b/roomserver/internal/input/input_missing.go @@ -613,12 +613,13 @@ func (t *missingStateReq) lookupMissingStateViaState( return nil, err } // Check that the returned state is valid. - if err := state.Check(ctx, roomVersion, t.keys, nil); err != nil { + authEvents, stateEvents, err := state.Check(ctx, roomVersion, t.keys, nil) + if err != nil { return nil, err } parsedState := &parsedRespState{ - AuthEvents: make([]*gomatrixserverlib.Event, len(state.AuthEvents)), - StateEvents: make([]*gomatrixserverlib.Event, len(state.StateEvents)), + AuthEvents: authEvents, + StateEvents: stateEvents, } // Cache the results of this state lookup and deduplicate anything we already // have in the cache, freeing up memory. From b49a6757c8a082f8d3048b53f1c160c73943fcee Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 5 Apr 2022 16:43:29 +0100 Subject: [PATCH 54/55] Update the "Are We Synapse Yet?" list --- are-we-synapse-yet.list | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/are-we-synapse-yet.list b/are-we-synapse-yet.list index c2c726e57..4281171ab 100644 --- a/are-we-synapse-yet.list +++ b/are-we-synapse-yet.list @@ -106,10 +106,13 @@ rst Users cannot set notifications powerlevel higher than their own (2 subtests) rst Both GET and PUT work rct POST /rooms/:room_id/receipt can create receipts red POST /rooms/:room_id/read_markers can create read marker +med POST /media/v3/upload can create an upload med POST /media/r0/upload can create an upload +med GET /media/v3/download can fetch the value again med GET /media/r0/download can fetch the value again cap GET /capabilities is present and well formed for registered user cap GET /r0/capabilities is not public +cap GET /v3/capabilities is not public reg Register with a recaptcha reg registration is idempotent, without username specified reg registration is idempotent, with username specified @@ -174,7 +177,9 @@ eph Ephemeral messages received from clients are correctly expired ali Room aliases can contain Unicode f,ali Remote room alias queries can handle Unicode ali Canonical alias can be set +ali Canonical alias can be set (3 subtests) ali Canonical alias can include alt_aliases +ali Canonical alias can include alt_aliases (4 subtests) ali Regular users can add and delete aliases in the default room configuration ali Regular users can add and delete aliases when m.room.aliases is restricted ali Deleting a non-existent alias should return a 404 @@ -478,6 +483,30 @@ rmv Inbound federation rejects invites which include invalid JSON for room versi rmv Outbound federation rejects invite response which include invalid JSON for room version 6 rmv Inbound federation rejects invite rejections which include invalid JSON for room version 6 rmv Server rejects invalid JSON in a version 6 room +rmv User can create and send/receive messages in a room with version 7 (2 subtests) +rmv local user can join room with version 7 +rmv User can invite local user to room with version 7 +rmv remote user can join room with version 7 +rmv User can invite remote user to room with version 7 +rmv Remote user can backfill in a room with version 7 +rmv Can reject invites over federation for rooms with version 7 +rmv Can receive redactions from regular users over federation in room version 7 +rmv User can create and send/receive messages in a room with version 8 (2 subtests) +rmv local user can join room with version 8 +rmv User can invite local user to room with version 8 +rmv remote user can join room with version 8 +rmv User can invite remote user to room with version 8 +rmv Remote user can backfill in a room with version 8 +rmv Can reject invites over federation for rooms with version 8 +rmv Can receive redactions from regular users over federation in room version 8 +rmv User can create and send/receive messages in a room with version 9 (2 subtests) +rmv local user can join room with version 9 +rmv User can invite local user to room with version 9 +rmv remote user can join room with version 9 +rmv User can invite remote user to room with version 9 +rmv Remote user can backfill in a room with version 9 +rmv Can reject invites over federation for rooms with version 9 +rmv Can receive redactions from regular users over federation in room version 9 pre Presence changes are reported to local room members f,pre Presence changes are also reported to remote room members pre Presence changes to UNAVAILABLE are reported to local room members @@ -772,12 +801,15 @@ app AS can make room aliases app Regular users cannot create room aliases within the AS namespace app AS-ghosted users can use rooms via AS app AS-ghosted users can use rooms themselves +app AS-ghosted users can use rooms via AS (2 subtests) +app AS-ghosted users can use rooms themselves (3 subtests) app Ghost user must register before joining room app AS can set avatar for ghosted users app AS can set displayname for ghosted users app AS can't set displayname for random users app Inviting an AS-hosted user asks the AS server app Accesing an AS-hosted room alias asks the AS server +app Accesing an AS-hosted room alias asks the AS server (2 subtests) app Events in rooms with AS-hosted room aliases are sent to AS server app AS user (not ghost) can join room without registering app AS user (not ghost) can join room without registering, with user_id query param @@ -789,6 +821,8 @@ app AS can publish rooms in their own list app AS and main public room lists are separate app AS can deactivate a user psh Test that a message is pushed +psh Test that a message is pushed (6 subtests) +psh Test that rejected pushers are removed. (4 subtests) psh Invites are pushed psh Rooms with names are correctly named in pushed psh Rooms with canonical alias are correctly named in pushed @@ -857,9 +891,12 @@ pre Left room members do not cause problems for presence crm Rooms can be created with an initial invite list (SYN-205) (1 subtests) typ Typing notifications don't leak ban Non-present room members cannot ban others +ban Non-present room members cannot ban others (3 subtests) psh Getting push rules doesn't corrupt the cache SYN-390 +psh Getting push rules doesn't corrupt the cache SYN-390 (3 subtests) inv Test that we can be reinvited to a room we created syn Multiple calls to /sync should not cause 500 errors +syn Multiple calls to /sync should not cause 500 errors (6 subtests) gst Guest user can call /events on another world_readable room (SYN-606) gst Real user can call /events on another world_readable room (SYN-606) gst Events come down the correct room From f7109de500d675855f9d63c1959f244d34096a4b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 5 Apr 2022 16:45:01 +0100 Subject: [PATCH 55/55] Nuke a couple of tests for now --- sytest-blacklist | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/sytest-blacklist b/sytest-blacklist index 6a3b88390..a4bdf372d 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -1,35 +1,46 @@ # Blacklisted until matrix-org/dendrite#862 is reverted due to Riot bug + Latest account data appears in v2 /sync # Blacklisted because we don't support ignores yet + Ignore invite in incremental sync # Relies on a rejected PL event which will never be accepted into the DAG -# Caused by https://github.com/matrix-org/sytest/pull/911 + +# Caused by + Outbound federation requests missing prev_events and then asks for /state_ids and resolves the state -# We don't implement lazy membership loading yet. +# We don't implement lazy membership loading yet + The only membership state included in a gapped incremental sync is for senders in the timeline # Blacklisted out of flakiness after #1479 + Invited user can reject local invite after originator leaves Invited user can reject invite for empty room If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes # Blacklisted due to flakiness + Forgotten room messages cannot be paginated # Blacklisted due to flakiness after #1774 + Local device key changes get to remote servers with correct prev_id # Flakey + Local device key changes appear in /keys/changes # we don't support groups + Remove group category Remove group role # Flakey + AS-ghosted users can use rooms themselves AS-ghosted users can use rooms via AS Events in rooms with AS-hosted room aliases are sent to AS server @@ -37,6 +48,12 @@ Inviting an AS-hosted user asks the AS server Accesing an AS-hosted room alias asks the AS server # Flakey, need additional investigation + Messages that notify from another user increment notification_count Messages that highlight from another user increment unread highlight count Notifications can be viewed with GET /notifications + +# More flakey + +If remote user leaves room we no longer receive device updates +Local device key changes get to remote servers