diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index 4c53a6137..fef351de2 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -109,6 +109,11 @@ jobs: uses: actions/setup-go@v3 with: go-version: ${{ matrix.go }} + - name: Set up gotestfmt + uses: gotesttools/gotestfmt-action@v2 + with: + # Optional: pass GITHUB_TOKEN to avoid rate limiting. + token: ${{ secrets.GITHUB_TOKEN }} - uses: actions/cache@v3 with: path: | @@ -117,7 +122,7 @@ jobs: key: ${{ runner.os }}-go${{ matrix.go }}-test-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-go${{ matrix.go }}-test- - - run: go test ./... + - run: go test -json -v ./... 2>&1 | gotestfmt env: POSTGRES_HOST: localhost POSTGRES_USER: postgres diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml new file mode 100644 index 000000000..c07917248 --- /dev/null +++ b/.github/workflows/schedules.yaml @@ -0,0 +1,128 @@ +name: Scheduled + +on: + schedule: + - cron: '0 0 * * *' # every day at midnight + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # run go test with different go versions + test: + timeout-minutes: 20 + name: Unit tests (Go ${{ matrix.go }}) + runs-on: ubuntu-latest + # Service containers to run with `container-job` + services: + # Label used to access the service container + postgres: + # Docker Hub image + image: postgres:13-alpine + # Provide the password for postgres + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: dendrite + ports: + # Maps tcp port 5432 on service container to the host + - 5432:5432 + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + strategy: + fail-fast: false + matrix: + go: ["1.18", "1.19"] + steps: + - uses: actions/checkout@v3 + - name: Setup go + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go }} + - uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go${{ matrix.go }}-test-race-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go${{ matrix.go }}-test-race- + - run: go test -race ./... + env: + POSTGRES_HOST: localhost + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: dendrite + + # Dummy step to gate other tests on without repeating the whole list + initial-tests-done: + name: Initial tests passed + needs: [test] + runs-on: ubuntu-latest + if: ${{ !cancelled() }} # Run this even if prior jobs were skipped + steps: + - name: Check initial tests passed + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} + + # run Sytest in different variations + sytest: + timeout-minutes: 60 + needs: initial-tests-done + name: "Sytest (${{ matrix.label }})" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - label: SQLite + + - label: SQLite, full HTTP APIs + api: full-http + + - label: PostgreSQL + postgres: postgres + + - label: PostgreSQL, full HTTP APIs + postgres: postgres + api: full-http + container: + image: matrixdotorg/sytest-dendrite:latest + volumes: + - ${{ github.workspace }}:/src + env: + POSTGRES: ${{ matrix.postgres && 1}} + API: ${{ matrix.api && 1 }} + SYTEST_BRANCH: ${{ github.head_ref }} + RACE_DETECTION: 1 + steps: + - uses: actions/checkout@v2 + - name: Run Sytest + run: /bootstrap.sh dendrite + working-directory: /src + - name: Summarise results.tap + if: ${{ always() }} + run: /sytest/scripts/tap_to_gha.pl /logs/results.tap + - name: Sytest List Maintenance + if: ${{ always() }} + run: /src/show-expected-fail-tests.sh /logs/results.tap /src/sytest-whitelist /src/sytest-blacklist + continue-on-error: true # not fatal + - name: Are We Synapse Yet? + if: ${{ always() }} + run: /src/are-we-synapse-yet.py /logs/results.tap -v + continue-on-error: true # not fatal + - name: Upload Sytest logs + uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }}) + path: | + /logs/results.tap + /logs/**/*.log* diff --git a/CHANGES.md b/CHANGES.md index eea2c3c7c..ba14dd07a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,43 @@ # Changelog +## Dendrite 0.10.5 (2022-10-31) + +### Features + +* It is now possible to use hCaptcha instead of reCAPTCHA for protecting registration +* A new `auto_join_rooms` configuration option has been added for automatically joining new users to a set of rooms +* A new `/_dendrite/admin/downloadState/{serverName}/{roomID}` endpoint has been added, which allows a server administrator to attempt to repair a room with broken room state by downloading a state snapshot from another federated server in the room + +### Fixes + +* Querying cross-signing keys for users should now be considerably faster +* A bug in state resolution where some events were not correctly selected for third-party invites has been fixed +* A bug in state resolution which could result in `not in room` event rejections has been fixed +* When accepting a DM invite, it should now be possible to see messages that were sent before the invite was accepted +* Claiming remote E2EE one-time keys has been refactored and should be more reliable now +* Various fixes have been made to the `/members` endpoint, which may help with E2EE reliability and clients rendering memberships +* A race condition in the federation API destination queues has been fixed when associating queued events with remote server destinations +* A bug in the sync API where too many events were selected resulting in high CPU usage has been fixed +* Configuring the avatar URL for the Server Notices user should work correctly now + +## Dendrite 0.10.4 (2022-10-21) + +### Features + +* Various tables belonging to the user API will be renamed so that they are namespaced with the `userapi_` prefix + * Note that, after upgrading to this version, you should not revert to an older version of Dendrite as the database changes **will not** be reverted automatically +* The backoff and retry behaviour in the federation API has been refactored and improved + +### Fixes + +* Private read receipt support is now advertised in the client `/versions` endpoint +* Private read receipts will now clear notification counts properly +* A bug where a false `leave` membership transition was inserted into the timeline after accepting an invite has been fixed +* Some panics caused by concurrent map writes in the key server have been fixed +* The sync API now calculates membership transitions from state deltas more accurately +* Transaction IDs are now scoped to endpoints, which should fix some bugs where transaction ID reuse could cause nonsensical cached responses from some endpoints +* The length of the `type`, `sender`, `state_key` and `room_id` fields in events are now verified by number of bytes rather than codepoints after a spec clarification, reverting a change made in Dendrite 0.9.6 + ## Dendrite 0.10.3 (2022-10-14) ### Features diff --git a/build/docker/Dockerfile.demo-yggdrasil b/build/docker/Dockerfile.demo-yggdrasil new file mode 100644 index 000000000..76bf35823 --- /dev/null +++ b/build/docker/Dockerfile.demo-yggdrasil @@ -0,0 +1,25 @@ +FROM docker.io/golang:1.19-alpine AS base + +RUN apk --update --no-cache add bash build-base + +WORKDIR /build + +COPY . /build + +RUN mkdir -p bin +RUN go build -trimpath -o bin/ ./cmd/dendrite-demo-yggdrasil +RUN go build -trimpath -o bin/ ./cmd/create-account +RUN go build -trimpath -o bin/ ./cmd/generate-keys + +FROM alpine:latest +LABEL org.opencontainers.image.title="Dendrite (Yggdrasil demo)" +LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" +LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" +LABEL org.opencontainers.image.licenses="Apache-2.0" + +COPY --from=base /build/bin/* /usr/bin/ + +VOLUME /etc/dendrite +WORKDIR /etc/dendrite + +ENTRYPOINT ["/usr/bin/dendrite-demo-yggdrasil"] diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 4a96e4bef..adb4e40a6 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -101,18 +101,46 @@ func (m *DendriteMonolith) SessionCount() int { return len(m.PineconeQUIC.Protocol("matrix").Sessions()) } -func (m *DendriteMonolith) RegisterNetworkInterface(name string, index int, mtu int, up bool, broadcast bool, loopback bool, pointToPoint bool, multicast bool, addrs string) { - m.PineconeMulticast.RegisterInterface(pineconeMulticast.InterfaceInfo{ - Name: name, - Index: index, - Mtu: mtu, - Up: up, - Broadcast: broadcast, - Loopback: loopback, - PointToPoint: pointToPoint, - Multicast: multicast, - Addrs: addrs, - }) +type InterfaceInfo struct { + Name string + Index int + Mtu int + Up bool + Broadcast bool + Loopback bool + PointToPoint bool + Multicast bool + Addrs string +} + +type InterfaceRetriever interface { + CacheCurrentInterfaces() int + GetCachedInterface(index int) *InterfaceInfo +} + +func (m *DendriteMonolith) RegisterNetworkCallback(intfCallback InterfaceRetriever) { + callback := func() []pineconeMulticast.InterfaceInfo { + count := intfCallback.CacheCurrentInterfaces() + intfs := []pineconeMulticast.InterfaceInfo{} + for i := 0; i < count; i++ { + iface := intfCallback.GetCachedInterface(i) + if iface != nil { + intfs = append(intfs, pineconeMulticast.InterfaceInfo{ + Name: iface.Name, + Index: iface.Index, + Mtu: iface.Mtu, + Up: iface.Up, + Broadcast: iface.Broadcast, + Loopback: iface.Loopback, + PointToPoint: iface.PointToPoint, + Multicast: iface.Multicast, + Addrs: iface.Addrs, + }) + } + } + return intfs + } + m.PineconeMulticast.RegisterNetworkCallback(callback) } func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) { diff --git a/build/scripts/Complement.Dockerfile b/build/scripts/Complement.Dockerfile index 9936c7416..14b28498b 100644 --- a/build/scripts/Complement.Dockerfile +++ b/build/scripts/Complement.Dockerfile @@ -1,6 +1,6 @@ #syntax=docker/dockerfile:1.2 -FROM golang:1.19-buster as build +FROM golang:1.18-stretch as build RUN apt-get update && apt-get install -y sqlite3 WORKDIR /build diff --git a/build/scripts/ComplementLocal.Dockerfile b/build/scripts/ComplementLocal.Dockerfile index 2b84c4798..3a019fc20 100644 --- a/build/scripts/ComplementLocal.Dockerfile +++ b/build/scripts/ComplementLocal.Dockerfile @@ -8,7 +8,7 @@ # # Use these mounts to make use of this dockerfile: # COMPLEMENT_HOST_MOUNTS='/your/local/dendrite:/dendrite:ro;/your/go/path:/go:ro' -FROM golang:1.19-buster +FROM golang:1.18-stretch RUN apt-get update && apt-get install -y sqlite3 ENV SERVER_NAME=localhost diff --git a/build/scripts/ComplementPostgres.Dockerfile b/build/scripts/ComplementPostgres.Dockerfile index 48af2339b..99f27abce 100644 --- a/build/scripts/ComplementPostgres.Dockerfile +++ b/build/scripts/ComplementPostgres.Dockerfile @@ -50,4 +50,4 @@ CMD /build/run_postgres.sh && ./generate-keys --keysize 1024 --server $SERVER_NA # Bump max_open_conns up here in the global database config sed -i 's/max_open_conns:.*$/max_open_conns: 1990/g' dendrite.yaml && \ cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \ - exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0} \ No newline at end of file + exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0} diff --git a/clientapi/auth/login_publickey_ethereum.go b/clientapi/auth/login_publickey_ethereum.go index 90de33d2b..33c0a16d4 100644 --- a/clientapi/auth/login_publickey_ethereum.go +++ b/clientapi/auth/login_publickey_ethereum.go @@ -67,7 +67,7 @@ func (pk LoginPublicKeyEthereum) GetType() string { } func (pk LoginPublicKeyEthereum) AccountExists(ctx context.Context) (string, *jsonerror.MatrixError) { - localPart, err := userutil.ParseUsernameParam(pk.UserId, &pk.config.Matrix.ServerName) + localPart, _, err := userutil.ParseUsernameParam(pk.UserId, pk.config.Matrix) if err != nil { // userId does not exist return "", jsonerror.Forbidden("the address is incorrect, or the account does not exist.") @@ -129,7 +129,7 @@ func (pk LoginPublicKeyEthereum) ValidateLoginResponse() (bool, *jsonerror.Matri } // Error if the chainId is not supported by the server. - if !contains(pk.config.PublicKeyAuthentication.Ethereum.ChainIDs, message.GetChainID()) { + if pk.config.PublicKeyAuthentication.Ethereum.GetChainID() != message.GetChainID() { return false, jsonerror.Forbidden("chainId") } @@ -156,12 +156,3 @@ func (pk LoginPublicKeyEthereum) verifyMessageUserId(message *siwe.Message) bool // one derived from the signed message. return pk.UserId == strings.ToLower(expectedUserId) } - -func contains(list []int, element int) bool { - for _, i := range list { - if i == element { - return true - } - } - return false -} diff --git a/clientapi/auth/login_publickey_ethereum_test.go b/clientapi/auth/login_publickey_ethereum_test.go index 73842f9a0..cd7db05b2 100644 --- a/clientapi/auth/login_publickey_ethereum_test.go +++ b/clientapi/auth/login_publickey_ethereum_test.go @@ -18,13 +18,14 @@ import ( "context" "fmt" "net/http" + "strconv" "strings" "testing" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/internal/mapsutil" "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/test" + testutil "github.com/matrix-org/dendrite/test" uapi "github.com/matrix-org/dendrite/userapi/api" "github.com/stretchr/testify/assert" ) @@ -35,19 +36,17 @@ type loginContext struct { } func createLoginContext(_ *testing.T) *loginContext { - chainIds := []int{4} - cfg := &config.ClientAPI{ Matrix: &config.Global{ - ServerName: test.TestServerName, + ServerName: testutil.TestServerName, }, Derived: &config.Derived{}, PasswordAuthenticationDisabled: true, PublicKeyAuthentication: config.PublicKeyAuthentication{ Ethereum: config.EthereumAuthConfig{ - Enabled: true, - Version: 1, - ChainIDs: chainIds, + Enabled: true, + Version: 1, + ConfigChainID: strconv.Itoa(testutil.EthereumTestNetworkId), }, }, } @@ -154,9 +153,9 @@ func TestLoginPublicKeyEthereum(t *testing.T) { var userAPI fakePublicKeyUserApi ctx := context.Background() loginContext := createLoginContext(t) - wallet, _ := test.CreateTestAccount() - message, _ := test.CreateEip4361TestMessage(wallet.PublicAddress) - signature, _ := test.SignMessage(message.String(), wallet.PrivateKey) + wallet, _ := testutil.CreateTestAccount() + message, _ := testutil.CreateEip4361TestMessage(wallet.PublicAddress) + signature, _ := testutil.SignMessage(message.String(), wallet.PrivateKey) sessionId := publicKeyTestSession( &ctx, loginContext.config, @@ -165,7 +164,7 @@ func TestLoginPublicKeyEthereum(t *testing.T) { ) // Escape \t and \n. Work around for marshalling and unmarshalling message. - msgStr := test.FromEip4361MessageToString(message) + msgStr := testutil.FromEip4361MessageToString(message) body := fmt.Sprintf(`{ "type": "m.login.publickey", "auth": { @@ -219,8 +218,8 @@ func TestLoginPublicKeyEthereumMissingSignature(t *testing.T) { var userAPI fakePublicKeyUserApi ctx := context.Background() loginContext := createLoginContext(t) - wallet, _ := test.CreateTestAccount() - message, _ := test.CreateEip4361TestMessage(wallet.PublicAddress) + wallet, _ := testutil.CreateTestAccount() + message, _ := testutil.CreateEip4361TestMessage(wallet.PublicAddress) sessionId := publicKeyTestSession( &ctx, loginContext.config, @@ -229,7 +228,7 @@ func TestLoginPublicKeyEthereumMissingSignature(t *testing.T) { ) // Escape \t and \n. Work around for marshalling and unmarshalling message. - msgStr := test.FromEip4361MessageToString(message) + msgStr := testutil.FromEip4361MessageToString(message) body := fmt.Sprintf(`{ "type": "m.login.publickey", "auth": { @@ -280,7 +279,7 @@ func TestLoginPublicKeyEthereumEmptyMessage(t *testing.T) { var userAPI fakePublicKeyUserApi ctx := context.Background() loginContext := createLoginContext(t) - wallet, _ := test.CreateTestAccount() + wallet, _ := testutil.CreateTestAccount() sessionId := publicKeyTestSession( &ctx, loginContext.config, @@ -333,7 +332,7 @@ func TestLoginPublicKeyEthereumWrongUserId(t *testing.T) { var userAPI fakePublicKeyUserApi ctx := context.Background() loginContext := createLoginContext(t) - wallet, _ := test.CreateTestAccount() + wallet, _ := testutil.CreateTestAccount() sessionId := publicKeyTestSession( &ctx, loginContext.config, diff --git a/clientapi/auth/login_publickey_test.go b/clientapi/auth/login_publickey_test.go index 6b95c5553..513616486 100644 --- a/clientapi/auth/login_publickey_test.go +++ b/clientapi/auth/login_publickey_test.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/setup/config" + testutil "github.com/matrix-org/dendrite/test" "github.com/stretchr/testify/assert" ) @@ -72,7 +73,10 @@ func TestLoginPublicKeyNewSession(t *testing.T) { params, "[object]") ethParams := params.(config.EthereumAuthParams) - assert.NotEmptyf(ethParams.ChainIDs, "ChainIDs actual: empty, expected not empty") + assert.Equalf( + testutil.EthereumTestNetworkId, + ethParams.ChainID, + "ChainID actual: %d, expected %d", ethParams.ChainID, testutil.EthereumTestNetworkId) assert.NotEmptyf(ethParams.Version, "Version actual: \"\", expected: not empty") } diff --git a/clientapi/auth/password.go b/clientapi/auth/password.go index 3bd77eb3d..1c8540e41 100644 --- a/clientapi/auth/password.go +++ b/clientapi/auth/password.go @@ -74,7 +74,7 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, JSON: jsonerror.BadJSON("A password must be supplied."), } } - localpart, err := userutil.ParseUsernameParam(username, &t.Config.Matrix.ServerName) + localpart, _, err := userutil.ParseUsernameParam(username, t.Config.Matrix) if err != nil { return nil, &util.JSONResponse{ Code: http.StatusUnauthorized, diff --git a/clientapi/authorization/authorization.go b/clientapi/authorization/authorization.go index f37f4becd..f81513d76 100644 --- a/clientapi/authorization/authorization.go +++ b/clientapi/authorization/authorization.go @@ -10,8 +10,8 @@ import ( func NewAuthorization(cfg *config.ClientAPI, rsAPI roomserver.ClientRoomserverAPI) authorization.Authorization { // Load authorization manager for Zion - if cfg.PublicKeyAuthentication.Ethereum.EnableAuthz { - auth, err := zion.NewZionAuthorization(rsAPI) + if cfg.PublicKeyAuthentication.Ethereum.GetEnableAuthZ() { + auth, err := zion.NewZionAuthorization(cfg, rsAPI) if err != nil { log.Errorln("Failed to initialise Zion authorization manager. Using default.", err) diff --git a/clientapi/routing/admin.go b/clientapi/routing/admin.go index 89c269f1a..9088f7716 100644 --- a/clientapi/routing/admin.go +++ b/clientapi/routing/admin.go @@ -70,7 +70,7 @@ func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, device *userapi if err != nil { return util.MessageResponse(http.StatusBadRequest, err.Error()) } - if domain != cfg.Matrix.ServerName { + if !cfg.Matrix.IsLocalServerName(domain) { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.MissingArgument("User ID must belong to this server."), @@ -169,7 +169,7 @@ func AdminMarkAsStale(req *http.Request, cfg *config.ClientAPI, keyAPI api.Clien if err != nil { return util.MessageResponse(http.StatusBadRequest, err.Error()) } - if domain == cfg.Matrix.ServerName { + if cfg.Matrix.IsLocalServerName(domain) { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.InvalidParam("Can not mark local device list as stale"), @@ -191,3 +191,43 @@ func AdminMarkAsStale(req *http.Request, cfg *config.ClientAPI, keyAPI api.Clien JSON: struct{}{}, } } + +func AdminDownloadState(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + roomID, ok := vars["roomID"] + if !ok { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.MissingArgument("Expecting room ID."), + } + } + serverName, ok := vars["serverName"] + if !ok { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.MissingArgument("Expecting remote server name."), + } + } + res := &roomserverAPI.PerformAdminDownloadStateResponse{} + if err := rsAPI.PerformAdminDownloadState( + req.Context(), + &roomserverAPI.PerformAdminDownloadStateRequest{ + UserID: device.UserID, + RoomID: roomID, + ServerName: gomatrixserverlib.ServerName(serverName), + }, + res, + ); err != nil { + return jsonerror.InternalAPIError(req.Context(), err) + } + if err := res.Error; err != nil { + return err.JSONResponse() + } + return util.JSONResponse{ + Code: 200, + JSON: map[string]interface{}{}, + } +} diff --git a/clientapi/routing/auth_fallback.go b/clientapi/routing/auth_fallback.go index abfe830fb..ad870993e 100644 --- a/clientapi/routing/auth_fallback.go +++ b/clientapi/routing/auth_fallback.go @@ -31,8 +31,7 @@ const recaptchaTemplate = ` Authentication - +