diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.md b/.github/ISSUE_TEMPLATE/BUG_REPORT.md index 206713e04..f40c56609 100644 --- a/.github/ISSUE_TEMPLATE/BUG_REPORT.md +++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.md @@ -7,24 +7,28 @@ about: Create a report to help us improve ### Background information -- **Dendrite version or git SHA**: -- **Monolith or Polylith?**: -- **SQLite3 or Postgres?**: -- **Running in Docker?**: +- **Dendrite version or git SHA**: +- **Monolith or Polylith?**: +- **SQLite3 or Postgres?**: +- **Running in Docker?**: - **`go version`**: - **Client used (if applicable)**: - ### Description - - **What** is the problem: - - **Who** is affected: - - **How** is this bug manifesting: - - **When** did this first appear: +- **What** is the problem: +- **Who** is affected: +- **How** is this bug manifesting: +- **When** did this first appear: + -* [ ] I have added tests for PR _or_ I have justified why this PR doesn't need tests. -* [ ] Pull request includes a [sign off](https://github.com/matrix-org/dendrite/blob/main/docs/CONTRIBUTING.md#sign-off) +* [ ] I have added Go unit tests or [Complement integration tests](https://github.com/matrix-org/complement) for this PR _or_ I have justified why this PR doesn't need tests +* [ ] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) _or_ I have already signed off privately Signed-off-by: `Your Name ` diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index 524d36039..4725637ed 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -109,6 +109,11 @@ jobs: uses: actions/setup-go@v3 with: go-version: ${{ matrix.go }} + - name: Set up gotestfmt + uses: gotesttools/gotestfmt-action@v2 + with: + # Optional: pass GITHUB_TOKEN to avoid rate limiting. + token: ${{ secrets.GITHUB_TOKEN }} - uses: actions/cache@v3 with: path: | @@ -117,7 +122,7 @@ jobs: key: ${{ runner.os }}-go${{ matrix.go }}-test-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-go${{ matrix.go }}-test- - - run: go test ./... + - run: go test -json -v ./... 2>&1 | gotestfmt env: POSTGRES_HOST: localhost POSTGRES_USER: postgres @@ -264,11 +269,18 @@ jobs: fail-fast: false matrix: include: - - label: SQLite + - label: SQLite native - - label: SQLite, full HTTP APIs + - label: SQLite Cgo + cgo: 1 + + - label: SQLite native, full HTTP APIs api: full-http + - label: SQLite Cgo, full HTTP APIs + api: full-http + cgo: 1 + - label: PostgreSQL postgres: postgres @@ -283,6 +295,7 @@ jobs: POSTGRES: ${{ matrix.postgres && 1}} API: ${{ matrix.api && 1 }} SYTEST_BRANCH: ${{ github.head_ref }} + CGO_ENABLED: ${{ matrix.cgo && 1 }} steps: - uses: actions/checkout@v3 - name: Run Sytest @@ -318,11 +331,18 @@ jobs: fail-fast: false matrix: include: - - label: SQLite + - label: SQLite native - - label: SQLite, full HTTP APIs + - label: SQLite Cgo + cgo: 1 + + - label: SQLite native, full HTTP APIs api: full-http + - label: SQLite Cgo, full HTTP APIs + api: full-http + cgo: 1 + - label: PostgreSQL postgres: Postgres @@ -342,7 +362,7 @@ jobs: # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64 run: | sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev - go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest + go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest - name: Run actions/checkout@v3 for dendrite uses: actions/checkout@v3 @@ -388,6 +408,7 @@ jobs: env: COMPLEMENT_BASE_IMAGE: complement-dendrite:latest API: ${{ matrix.api && 1 }} + CGO_ENABLED: ${{ matrix.cgo && 1 }} working-directory: complement integration-tests-done: diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml new file mode 100644 index 000000000..c07917248 --- /dev/null +++ b/.github/workflows/schedules.yaml @@ -0,0 +1,128 @@ +name: Scheduled + +on: + schedule: + - cron: '0 0 * * *' # every day at midnight + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # run go test with different go versions + test: + timeout-minutes: 20 + name: Unit tests (Go ${{ matrix.go }}) + runs-on: ubuntu-latest + # Service containers to run with `container-job` + services: + # Label used to access the service container + postgres: + # Docker Hub image + image: postgres:13-alpine + # Provide the password for postgres + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: dendrite + ports: + # Maps tcp port 5432 on service container to the host + - 5432:5432 + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + strategy: + fail-fast: false + matrix: + go: ["1.18", "1.19"] + steps: + - uses: actions/checkout@v3 + - name: Setup go + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go }} + - uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go${{ matrix.go }}-test-race-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go${{ matrix.go }}-test-race- + - run: go test -race ./... + env: + POSTGRES_HOST: localhost + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: dendrite + + # Dummy step to gate other tests on without repeating the whole list + initial-tests-done: + name: Initial tests passed + needs: [test] + runs-on: ubuntu-latest + if: ${{ !cancelled() }} # Run this even if prior jobs were skipped + steps: + - name: Check initial tests passed + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} + + # run Sytest in different variations + sytest: + timeout-minutes: 60 + needs: initial-tests-done + name: "Sytest (${{ matrix.label }})" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - label: SQLite + + - label: SQLite, full HTTP APIs + api: full-http + + - label: PostgreSQL + postgres: postgres + + - label: PostgreSQL, full HTTP APIs + postgres: postgres + api: full-http + container: + image: matrixdotorg/sytest-dendrite:latest + volumes: + - ${{ github.workspace }}:/src + env: + POSTGRES: ${{ matrix.postgres && 1}} + API: ${{ matrix.api && 1 }} + SYTEST_BRANCH: ${{ github.head_ref }} + RACE_DETECTION: 1 + steps: + - uses: actions/checkout@v2 + - name: Run Sytest + run: /bootstrap.sh dendrite + working-directory: /src + - name: Summarise results.tap + if: ${{ always() }} + run: /sytest/scripts/tap_to_gha.pl /logs/results.tap + - name: Sytest List Maintenance + if: ${{ always() }} + run: /src/show-expected-fail-tests.sh /logs/results.tap /src/sytest-whitelist /src/sytest-blacklist + continue-on-error: true # not fatal + - name: Are We Synapse Yet? + if: ${{ always() }} + run: /src/are-we-synapse-yet.py /logs/results.tap -v + continue-on-error: true # not fatal + - name: Upload Sytest logs + uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }}) + path: | + /logs/results.tap + /logs/**/*.log* diff --git a/CHANGES.md b/CHANGES.md index dbe2ccf02..55df36f96 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,96 @@ # Changelog +## Dendrite 0.10.6 (2022-11-01) + +### Features + +* History visibility checks have been optimised, which should speed up response times on a variety of endpoints (including `/sync`, `/messages`, `/context` and others) and reduce database load +* The built-in NATS Server has been updated to version 2.9.4 +* Some other minor dependencies have been updated + +### Fixes + +* A panic has been fixed in the sync API PDU stream which could cause requests to fail +* The `/members` response now contains the `room_id` field, which may fix some E2EE problems with clients using the JS SDK (contributed by [ashkitten](https://github.com/ashkitten)) +* The auth difference calculation in state resolution v2 has been tweaked for clarity (and moved into gomatrixserverlib with the rest of the state resolution code) + +## Dendrite 0.10.5 (2022-10-31) + +### Features + +* It is now possible to use hCaptcha instead of reCAPTCHA for protecting registration +* A new `auto_join_rooms` configuration option has been added for automatically joining new users to a set of rooms +* A new `/_dendrite/admin/downloadState/{serverName}/{roomID}` endpoint has been added, which allows a server administrator to attempt to repair a room with broken room state by downloading a state snapshot from another federated server in the room + +### Fixes + +* Querying cross-signing keys for users should now be considerably faster +* A bug in state resolution where some events were not correctly selected for third-party invites has been fixed +* A bug in state resolution which could result in `not in room` event rejections has been fixed +* When accepting a DM invite, it should now be possible to see messages that were sent before the invite was accepted +* Claiming remote E2EE one-time keys has been refactored and should be more reliable now +* Various fixes have been made to the `/members` endpoint, which may help with E2EE reliability and clients rendering memberships +* A race condition in the federation API destination queues has been fixed when associating queued events with remote server destinations +* A bug in the sync API where too many events were selected resulting in high CPU usage has been fixed +* Configuring the avatar URL for the Server Notices user should work correctly now + +## Dendrite 0.10.4 (2022-10-21) + +### Features + +* Various tables belonging to the user API will be renamed so that they are namespaced with the `userapi_` prefix + * Note that, after upgrading to this version, you should not revert to an older version of Dendrite as the database changes **will not** be reverted automatically +* The backoff and retry behaviour in the federation API has been refactored and improved + +### Fixes + +* Private read receipt support is now advertised in the client `/versions` endpoint +* Private read receipts will now clear notification counts properly +* A bug where a false `leave` membership transition was inserted into the timeline after accepting an invite has been fixed +* Some panics caused by concurrent map writes in the key server have been fixed +* The sync API now calculates membership transitions from state deltas more accurately +* Transaction IDs are now scoped to endpoints, which should fix some bugs where transaction ID reuse could cause nonsensical cached responses from some endpoints +* The length of the `type`, `sender`, `state_key` and `room_id` fields in events are now verified by number of bytes rather than codepoints after a spec clarification, reverting a change made in Dendrite 0.9.6 + +## Dendrite 0.10.3 (2022-10-14) + +### Features + +* Event relations are now tracked and support for the `/room/{roomID}/relations/...` client API endpoints have been added +* Support has been added for private read receipts +* The built-in NATS Server has been updated to version 2.9.3 + +### Fixes + +* The `unread_notifications` are now always populated in joined room responses +* The `/get_missing_events` federation API endpoint should now work correctly for rooms with `joined` and `invited` visibility settings, returning redacted events for events that other servers are not allowed to see +* The `/event` client API endpoint now applies history visibility correctly +* Read markers should now be updated much more reliably +* A rare bug in the sync API which could cause some `join` memberships to be incorrectly overwritten by other memberships when working out which rooms to populate has been fixed +* The federation API now correctly updates the joined hosts table during a state rewrite + +## Dendrite 0.10.2 (2022-10-07) + +### Features + +* Dendrite will now fail to start if there is an obvious problem with the configured `max_open_conns` when using PostgreSQL database backends, since this can lead to instability and performance issues + * More information on this is available [in the documentation](https://matrix-org.github.io/dendrite/installation/start/optimisation#postgresql-connection-limit) +* Unnecessary/empty fields will no longer be sent in `/sync` responses +* It is now possible to configure `old_private_keys` from previous Matrix installations on the same domain if only public key is known, to make it easier to expire old keys correctly + * You can configure either just the `private_key` path, or you can supply both the `public_key` and `key_id` + +### Fixes + +* The sync transaction behaviour has been modified further so that errors in one stream should not propagate to other streams unnecessarily +* Rooms should now be classified as DM rooms correctly by passing through `is_direct` and unsigned hints +* A bug which caused marking device lists as stale to consume lots of CPU has been fixed +* Users accepting invites should no longer cause unnecessary federated joins if there are already other local users in the room +* The sync API state range queries have been optimised by adding missing indexes +* It should now be possible to configure non-English languages for full-text search in `search.language` +* The roomserver will no longer attempt to perform federated requests to the local server when trying to fetch missing events +* The `/keys/upload` endpoint will now always return the `one_time_keys_counts`, which may help with E2EE reliability +* The sync API will now retrieve the latest stream position before processing each stream rather than at the beginning of the request, to hopefully reduce the number of round-trips to `/sync` + ## Dendrite 0.10.1 (2022-09-30) ### Features diff --git a/README.md b/README.md index 3bb9a2350..dfef11bae 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ $ ./bin/dendrite-monolith-server --tls-cert server.crt --tls-key server.key --co # Create an user account (add -admin for an admin user). # Specify the localpart only, e.g. 'alice' for '@alice:domain.com' -$ ./bin/create-account --config dendrite.yaml --url http://localhost:8008 --username alice +$ ./bin/create-account --config dendrite.yaml --username alice ``` Then point your favourite Matrix client at `http://localhost:8008` or `https://localhost:8448`. @@ -90,7 +90,7 @@ We use a script called Are We Synapse Yet which checks Sytest compliance rates. test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it updates with CI. As of August 2022 we're at around 90% CS API coverage and 95% Federation coverage, though check CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse -servers such as matrix.org reasonably well, although there are still some missing features (like Search). +servers such as matrix.org reasonably well, although there are still some missing features (like SSO and Third-party ID APIs). We are prioritising features that will benefit single-user homeservers first (e.g Receipts, E2E) rather than features that massive deployments may be interested in (OpenID, Guests, Admin APIs, AS API). @@ -112,6 +112,7 @@ This means Dendrite supports amongst others: - Guests - User Directory - Presence +- Fulltext search ## Contributing diff --git a/appservice/api/query.go b/appservice/api/query.go index 4d1cf9474..eb567b2ee 100644 --- a/appservice/api/query.go +++ b/appservice/api/query.go @@ -19,11 +19,13 @@ package api import ( "context" + "encoding/json" "errors" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" ) // AppServiceInternalAPI is used to query user and room alias data from application @@ -41,6 +43,10 @@ type AppServiceInternalAPI interface { req *UserIDExistsRequest, resp *UserIDExistsResponse, ) error + + Locations(ctx context.Context, req *LocationRequest, resp *LocationResponse) error + User(ctx context.Context, request *UserRequest, response *UserResponse) error + Protocols(ctx context.Context, req *ProtocolRequest, resp *ProtocolResponse) error } // RoomAliasExistsRequest is a request to an application service @@ -77,6 +83,73 @@ type UserIDExistsResponse struct { UserIDExists bool `json:"exists"` } +const ( + ASProtocolPath = "/_matrix/app/unstable/thirdparty/protocol/" + ASUserPath = "/_matrix/app/unstable/thirdparty/user" + ASLocationPath = "/_matrix/app/unstable/thirdparty/location" +) + +type ProtocolRequest struct { + Protocol string `json:"protocol,omitempty"` +} + +type ProtocolResponse struct { + Protocols map[string]ASProtocolResponse `json:"protocols"` + Exists bool `json:"exists"` +} + +type ASProtocolResponse struct { + FieldTypes map[string]FieldType `json:"field_types,omitempty"` // NOTSPEC: field_types is required by the spec + Icon string `json:"icon"` + Instances []ProtocolInstance `json:"instances"` + LocationFields []string `json:"location_fields"` + UserFields []string `json:"user_fields"` +} + +type FieldType struct { + Placeholder string `json:"placeholder"` + Regexp string `json:"regexp"` +} + +type ProtocolInstance struct { + Description string `json:"desc"` + Icon string `json:"icon,omitempty"` + NetworkID string `json:"network_id,omitempty"` // NOTSPEC: network_id is required by the spec + Fields json.RawMessage `json:"fields,omitempty"` // NOTSPEC: fields is required by the spec +} + +type UserRequest struct { + Protocol string `json:"protocol"` + Params string `json:"params"` +} + +type UserResponse struct { + Users []ASUserResponse `json:"users,omitempty"` + Exists bool `json:"exists,omitempty"` +} + +type ASUserResponse struct { + Protocol string `json:"protocol"` + UserID string `json:"userid"` + Fields json.RawMessage `json:"fields"` +} + +type LocationRequest struct { + Protocol string `json:"protocol"` + Params string `json:"params"` +} + +type LocationResponse struct { + Locations []ASLocationResponse `json:"locations,omitempty"` + Exists bool `json:"exists,omitempty"` +} + +type ASLocationResponse struct { + Alias string `json:"alias"` + Protocol string `json:"protocol"` + Fields json.RawMessage `json:"fields"` +} + // RetrieveUserProfile is a wrapper that queries both the local database and // application services for a given user's profile // TODO: Remove this, it's called from federationapi and clientapi but is a pure function diff --git a/appservice/appservice.go b/appservice/appservice.go index 9000adb1d..0c778b6ca 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -18,6 +18,7 @@ import ( "context" "crypto/tls" "net/http" + "sync" "time" "github.com/gorilla/mux" @@ -58,8 +59,10 @@ func NewInternalAPI( // Create appserivce query API with an HTTP client that will be used for all // outbound and inbound requests (inbound only for the internal API) appserviceQueryAPI := &query.AppServiceQueryAPI{ - HTTPClient: client, - Cfg: &base.Cfg.AppServiceAPI, + HTTPClient: client, + Cfg: &base.Cfg.AppServiceAPI, + ProtocolCache: map[string]appserviceAPI.ASProtocolResponse{}, + CacheMu: sync.Mutex{}, } if len(base.Cfg.Derived.ApplicationServices) == 0 { diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index d44f32b38..ac68f4bd4 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -101,6 +101,11 @@ func (s *OutputRoomEventConsumer) onMessage( log.WithField("appservice", state.ID).Tracef("Appservice worker received %d message(s) from roomserver", len(msgs)) events := make([]*gomatrixserverlib.HeaderedEvent, 0, len(msgs)) for _, msg := range msgs { + // Only handle events we care about + receivedType := api.OutputType(msg.Header.Get(jetstream.RoomEventType)) + if receivedType != api.OutputTypeNewRoomEvent && receivedType != api.OutputTypeNewInviteEvent { + continue + } // Parse out the event JSON var output api.OutputEvent if err := json.Unmarshal(msg.Data, &output); err != nil { diff --git a/appservice/inthttp/client.go b/appservice/inthttp/client.go index 3ae2c9278..f7f164877 100644 --- a/appservice/inthttp/client.go +++ b/appservice/inthttp/client.go @@ -13,6 +13,9 @@ import ( const ( AppServiceRoomAliasExistsPath = "/appservice/RoomAliasExists" AppServiceUserIDExistsPath = "/appservice/UserIDExists" + AppServiceLocationsPath = "/appservice/locations" + AppServiceUserPath = "/appservice/users" + AppServiceProtocolsPath = "/appservice/protocols" ) // httpAppServiceQueryAPI contains the URL to an appservice query API and a @@ -58,3 +61,24 @@ func (h *httpAppServiceQueryAPI) UserIDExists( h.httpClient, ctx, request, response, ) } + +func (h *httpAppServiceQueryAPI) Locations(ctx context.Context, request *api.LocationRequest, response *api.LocationResponse) error { + return httputil.CallInternalRPCAPI( + "ASLocation", h.appserviceURL+AppServiceLocationsPath, + h.httpClient, ctx, request, response, + ) +} + +func (h *httpAppServiceQueryAPI) User(ctx context.Context, request *api.UserRequest, response *api.UserResponse) error { + return httputil.CallInternalRPCAPI( + "ASUser", h.appserviceURL+AppServiceUserPath, + h.httpClient, ctx, request, response, + ) +} + +func (h *httpAppServiceQueryAPI) Protocols(ctx context.Context, request *api.ProtocolRequest, response *api.ProtocolResponse) error { + return httputil.CallInternalRPCAPI( + "ASProtocols", h.appserviceURL+AppServiceProtocolsPath, + h.httpClient, ctx, request, response, + ) +} diff --git a/appservice/inthttp/server.go b/appservice/inthttp/server.go index 01d9f9895..ccf5c83d8 100644 --- a/appservice/inthttp/server.go +++ b/appservice/inthttp/server.go @@ -2,6 +2,7 @@ package inthttp import ( "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/internal/httputil" ) @@ -17,4 +18,19 @@ func AddRoutes(a api.AppServiceInternalAPI, internalAPIMux *mux.Router) { AppServiceUserIDExistsPath, httputil.MakeInternalRPCAPI("AppserviceUserIDExists", a.UserIDExists), ) + + internalAPIMux.Handle( + AppServiceProtocolsPath, + httputil.MakeInternalRPCAPI("AppserviceProtocols", a.Protocols), + ) + + internalAPIMux.Handle( + AppServiceLocationsPath, + httputil.MakeInternalRPCAPI("AppserviceLocations", a.Locations), + ) + + internalAPIMux.Handle( + AppServiceUserPath, + httputil.MakeInternalRPCAPI("AppserviceUser", a.User), + ) } diff --git a/appservice/query/query.go b/appservice/query/query.go index 53b34cb18..2348eab4b 100644 --- a/appservice/query/query.go +++ b/appservice/query/query.go @@ -18,13 +18,18 @@ package query import ( "context" + "encoding/json" + "io" "net/http" "net/url" + "strings" + "sync" + + "github.com/opentracing/opentracing-go" + log "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/setup/config" - opentracing "github.com/opentracing/opentracing-go" - log "github.com/sirupsen/logrus" ) const roomAliasExistsPath = "/rooms/" @@ -32,8 +37,10 @@ const userIDExistsPath = "/users/" // AppServiceQueryAPI is an implementation of api.AppServiceQueryAPI type AppServiceQueryAPI struct { - HTTPClient *http.Client - Cfg *config.AppServiceAPI + HTTPClient *http.Client + Cfg *config.AppServiceAPI + ProtocolCache map[string]api.ASProtocolResponse + CacheMu sync.Mutex } // RoomAliasExists performs a request to '/room/{roomAlias}' on all known @@ -165,3 +172,178 @@ func (a *AppServiceQueryAPI) UserIDExists( response.UserIDExists = false return nil } + +type thirdpartyResponses interface { + api.ASProtocolResponse | []api.ASUserResponse | []api.ASLocationResponse +} + +func requestDo[T thirdpartyResponses](client *http.Client, url string, response *T) (err error) { + origURL := url + // try v1 and unstable appservice endpoints + for _, version := range []string{"v1", "unstable"} { + var resp *http.Response + var body []byte + asURL := strings.Replace(origURL, "unstable", version, 1) + resp, err = client.Get(asURL) + if err != nil { + continue + } + defer resp.Body.Close() // nolint: errcheck + body, err = io.ReadAll(resp.Body) + if err != nil { + continue + } + return json.Unmarshal(body, &response) + } + return err +} + +func (a *AppServiceQueryAPI) Locations( + ctx context.Context, + req *api.LocationRequest, + resp *api.LocationResponse, +) error { + params, err := url.ParseQuery(req.Params) + if err != nil { + return err + } + + for _, as := range a.Cfg.Derived.ApplicationServices { + var asLocations []api.ASLocationResponse + params.Set("access_token", as.HSToken) + + url := as.URL + api.ASLocationPath + if req.Protocol != "" { + url += "/" + req.Protocol + } + + if err := requestDo[[]api.ASLocationResponse](a.HTTPClient, url+"?"+params.Encode(), &asLocations); err != nil { + log.WithError(err).Error("unable to get 'locations' from application service") + continue + } + + resp.Locations = append(resp.Locations, asLocations...) + } + + if len(resp.Locations) == 0 { + resp.Exists = false + return nil + } + resp.Exists = true + return nil +} + +func (a *AppServiceQueryAPI) User( + ctx context.Context, + req *api.UserRequest, + resp *api.UserResponse, +) error { + params, err := url.ParseQuery(req.Params) + if err != nil { + return err + } + + for _, as := range a.Cfg.Derived.ApplicationServices { + var asUsers []api.ASUserResponse + params.Set("access_token", as.HSToken) + + url := as.URL + api.ASUserPath + if req.Protocol != "" { + url += "/" + req.Protocol + } + + if err := requestDo[[]api.ASUserResponse](a.HTTPClient, url+"?"+params.Encode(), &asUsers); err != nil { + log.WithError(err).Error("unable to get 'user' from application service") + continue + } + + resp.Users = append(resp.Users, asUsers...) + } + + if len(resp.Users) == 0 { + resp.Exists = false + return nil + } + resp.Exists = true + return nil +} + +func (a *AppServiceQueryAPI) Protocols( + ctx context.Context, + req *api.ProtocolRequest, + resp *api.ProtocolResponse, +) error { + + // get a single protocol response + if req.Protocol != "" { + + a.CacheMu.Lock() + defer a.CacheMu.Unlock() + if proto, ok := a.ProtocolCache[req.Protocol]; ok { + resp.Exists = true + resp.Protocols = map[string]api.ASProtocolResponse{ + req.Protocol: proto, + } + return nil + } + + response := api.ASProtocolResponse{} + for _, as := range a.Cfg.Derived.ApplicationServices { + var proto api.ASProtocolResponse + if err := requestDo[api.ASProtocolResponse](a.HTTPClient, as.URL+api.ASProtocolPath+req.Protocol, &proto); err != nil { + log.WithError(err).Error("unable to get 'protocol' from application service") + continue + } + + if len(response.Instances) != 0 { + response.Instances = append(response.Instances, proto.Instances...) + } else { + response = proto + } + } + + if len(response.Instances) == 0 { + resp.Exists = false + return nil + } + + resp.Exists = true + resp.Protocols = map[string]api.ASProtocolResponse{ + req.Protocol: response, + } + a.ProtocolCache[req.Protocol] = response + return nil + } + + response := make(map[string]api.ASProtocolResponse, len(a.Cfg.Derived.ApplicationServices)) + + for _, as := range a.Cfg.Derived.ApplicationServices { + for _, p := range as.Protocols { + var proto api.ASProtocolResponse + if err := requestDo[api.ASProtocolResponse](a.HTTPClient, as.URL+api.ASProtocolPath+p, &proto); err != nil { + log.WithError(err).Error("unable to get 'protocol' from application service") + continue + } + existing, ok := response[p] + if !ok { + response[p] = proto + continue + } + existing.Instances = append(existing.Instances, proto.Instances...) + response[p] = existing + } + } + + if len(response) == 0 { + resp.Exists = false + return nil + } + + a.CacheMu.Lock() + defer a.CacheMu.Unlock() + a.ProtocolCache = response + + resp.Exists = true + resp.Protocols = response + return nil +} diff --git a/are-we-synapse-yet.list b/are-we-synapse-yet.list index c776a7400..81c0f8049 100644 --- a/are-we-synapse-yet.list +++ b/are-we-synapse-yet.list @@ -643,7 +643,7 @@ fed Inbound federation redacts events from erased users fme Outbound federation can request missing events fme Inbound federation can return missing events for world_readable visibility fme Inbound federation can return missing events for shared visibility -fme Inbound federation can return missing events for invite visibility +fme Inbound federation can return missing events for invited visibility fme Inbound federation can return missing events for joined visibility fme outliers whose auth_events are in a different room are correctly rejected fbk Outbound federation can backfill events diff --git a/build/docker/Dockerfile.demo-yggdrasil b/build/docker/Dockerfile.demo-yggdrasil new file mode 100644 index 000000000..76bf35823 --- /dev/null +++ b/build/docker/Dockerfile.demo-yggdrasil @@ -0,0 +1,25 @@ +FROM docker.io/golang:1.19-alpine AS base + +RUN apk --update --no-cache add bash build-base + +WORKDIR /build + +COPY . /build + +RUN mkdir -p bin +RUN go build -trimpath -o bin/ ./cmd/dendrite-demo-yggdrasil +RUN go build -trimpath -o bin/ ./cmd/create-account +RUN go build -trimpath -o bin/ ./cmd/generate-keys + +FROM alpine:latest +LABEL org.opencontainers.image.title="Dendrite (Yggdrasil demo)" +LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" +LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" +LABEL org.opencontainers.image.licenses="Apache-2.0" + +COPY --from=base /build/bin/* /usr/bin/ + +VOLUME /etc/dendrite +WORKDIR /etc/dendrite + +ENTRYPOINT ["/usr/bin/dendrite-demo-yggdrasil"] diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 4a96e4bef..adb4e40a6 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -101,18 +101,46 @@ func (m *DendriteMonolith) SessionCount() int { return len(m.PineconeQUIC.Protocol("matrix").Sessions()) } -func (m *DendriteMonolith) RegisterNetworkInterface(name string, index int, mtu int, up bool, broadcast bool, loopback bool, pointToPoint bool, multicast bool, addrs string) { - m.PineconeMulticast.RegisterInterface(pineconeMulticast.InterfaceInfo{ - Name: name, - Index: index, - Mtu: mtu, - Up: up, - Broadcast: broadcast, - Loopback: loopback, - PointToPoint: pointToPoint, - Multicast: multicast, - Addrs: addrs, - }) +type InterfaceInfo struct { + Name string + Index int + Mtu int + Up bool + Broadcast bool + Loopback bool + PointToPoint bool + Multicast bool + Addrs string +} + +type InterfaceRetriever interface { + CacheCurrentInterfaces() int + GetCachedInterface(index int) *InterfaceInfo +} + +func (m *DendriteMonolith) RegisterNetworkCallback(intfCallback InterfaceRetriever) { + callback := func() []pineconeMulticast.InterfaceInfo { + count := intfCallback.CacheCurrentInterfaces() + intfs := []pineconeMulticast.InterfaceInfo{} + for i := 0; i < count; i++ { + iface := intfCallback.GetCachedInterface(i) + if iface != nil { + intfs = append(intfs, pineconeMulticast.InterfaceInfo{ + Name: iface.Name, + Index: iface.Index, + Mtu: iface.Mtu, + Up: iface.Up, + Broadcast: iface.Broadcast, + Loopback: iface.Loopback, + PointToPoint: iface.PointToPoint, + Multicast: iface.Multicast, + Addrs: iface.Addrs, + }) + } + } + return intfs + } + m.PineconeMulticast.RegisterNetworkCallback(callback) } func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) { diff --git a/clientapi/auth/password.go b/clientapi/auth/password.go index bcb4ca97b..700a72f5d 100644 --- a/clientapi/auth/password.go +++ b/clientapi/auth/password.go @@ -68,7 +68,13 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, JSON: jsonerror.BadJSON("A username must be supplied."), } } - localpart, err := userutil.ParseUsernameParam(username, &t.Config.Matrix.ServerName) + if len(r.Password) == 0 { + return nil, &util.JSONResponse{ + Code: http.StatusUnauthorized, + JSON: jsonerror.BadJSON("A password must be supplied."), + } + } + localpart, _, err := userutil.ParseUsernameParam(username, t.Config.Matrix) if err != nil { return nil, &util.JSONResponse{ Code: http.StatusUnauthorized, diff --git a/clientapi/routing/account_data.go b/clientapi/routing/account_data.go index b28f0bb1f..4742b1240 100644 --- a/clientapi/routing/account_data.go +++ b/clientapi/routing/account_data.go @@ -154,33 +154,31 @@ func SaveReadMarker( return *resErr } - if r.FullyRead == "" { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON("Missing m.fully_read mandatory field"), + if r.FullyRead != "" { + data, err := json.Marshal(fullyReadEvent{EventID: r.FullyRead}) + if err != nil { + return jsonerror.InternalServerError() + } + + dataReq := api.InputAccountDataRequest{ + UserID: device.UserID, + DataType: "m.fully_read", + RoomID: roomID, + AccountData: data, + } + dataRes := api.InputAccountDataResponse{} + if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed") + return util.ErrorResponse(err) } } - data, err := json.Marshal(fullyReadEvent{EventID: r.FullyRead}) - if err != nil { - return jsonerror.InternalServerError() - } - - dataReq := api.InputAccountDataRequest{ - UserID: device.UserID, - DataType: "m.fully_read", - RoomID: roomID, - AccountData: data, - } - dataRes := api.InputAccountDataResponse{} - if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed") - return util.ErrorResponse(err) - } - - // Handle the read receipt that may be included in the read marker + // Handle the read receipts that may be included in the read marker. if r.Read != "" { - return SetReceipt(req, syncProducer, device, roomID, "m.read", r.Read) + return SetReceipt(req, userAPI, syncProducer, device, roomID, "m.read", r.Read) + } + if r.ReadPrivate != "" { + return SetReceipt(req, userAPI, syncProducer, device, roomID, "m.read.private", r.ReadPrivate) } return util.JSONResponse{ diff --git a/clientapi/routing/admin.go b/clientapi/routing/admin.go index 89c269f1a..9088f7716 100644 --- a/clientapi/routing/admin.go +++ b/clientapi/routing/admin.go @@ -70,7 +70,7 @@ func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, device *userapi if err != nil { return util.MessageResponse(http.StatusBadRequest, err.Error()) } - if domain != cfg.Matrix.ServerName { + if !cfg.Matrix.IsLocalServerName(domain) { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.MissingArgument("User ID must belong to this server."), @@ -169,7 +169,7 @@ func AdminMarkAsStale(req *http.Request, cfg *config.ClientAPI, keyAPI api.Clien if err != nil { return util.MessageResponse(http.StatusBadRequest, err.Error()) } - if domain == cfg.Matrix.ServerName { + if cfg.Matrix.IsLocalServerName(domain) { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.InvalidParam("Can not mark local device list as stale"), @@ -191,3 +191,43 @@ func AdminMarkAsStale(req *http.Request, cfg *config.ClientAPI, keyAPI api.Clien JSON: struct{}{}, } } + +func AdminDownloadState(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + roomID, ok := vars["roomID"] + if !ok { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.MissingArgument("Expecting room ID."), + } + } + serverName, ok := vars["serverName"] + if !ok { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.MissingArgument("Expecting remote server name."), + } + } + res := &roomserverAPI.PerformAdminDownloadStateResponse{} + if err := rsAPI.PerformAdminDownloadState( + req.Context(), + &roomserverAPI.PerformAdminDownloadStateRequest{ + UserID: device.UserID, + RoomID: roomID, + ServerName: gomatrixserverlib.ServerName(serverName), + }, + res, + ); err != nil { + return jsonerror.InternalAPIError(req.Context(), err) + } + if err := res.Error; err != nil { + return err.JSONResponse() + } + return util.JSONResponse{ + Code: 200, + JSON: map[string]interface{}{}, + } +} diff --git a/clientapi/routing/auth_fallback.go b/clientapi/routing/auth_fallback.go index abfe830fb..ad870993e 100644 --- a/clientapi/routing/auth_fallback.go +++ b/clientapi/routing/auth_fallback.go @@ -31,8 +31,7 @@ const recaptchaTemplate = ` Authentication - +