From ed8b5d09eb8df9dec0358fab6fbd38643f82efb8 Mon Sep 17 00:00:00 2001 From: Tak Wai Wong <64229756+tak-hntlabs@users.noreply.github.com> Date: Fri, 28 Oct 2022 14:43:21 -0700 Subject: [PATCH] refresh fork with upstream --- .github/workflows/schedules.yaml | 128 ++++++++++++++++++ build/docker/Dockerfile.demo-yggdrasil | 25 ++++ build/gobind-pinecone/monolith.go | 52 +++++-- clientapi/auth/login_publickey_ethereum.go | 2 +- clientapi/auth/password.go | 2 +- clientapi/routing/admin.go | 4 +- clientapi/routing/auth_fallback.go | 18 +-- clientapi/routing/createroom.go | 25 +++- clientapi/routing/directory.go | 49 ++++++- clientapi/routing/directory_public.go | 43 ++++-- clientapi/routing/directory_public_test.go | 2 +- clientapi/routing/login.go | 6 +- clientapi/routing/membership.go | 9 +- clientapi/routing/openid.go | 2 +- clientapi/routing/profile.go | 34 ++++- clientapi/routing/redaction.go | 3 +- clientapi/routing/register.go | 6 +- clientapi/routing/routing.go | 23 +++- clientapi/routing/sendevent.go | 5 +- clientapi/threepid/invites.go | 2 +- clientapi/userutil/userutil.go | 13 +- clientapi/userutil/userutil_test.go | 25 +++- cmd/dendrite-demo-yggdrasil/yggconn/node.go | 86 ++++++------ dendrite-sample.monolith.yaml | 16 ++- dendrite-sample.polylith.yaml | 16 ++- federationapi/consumers/keychange.go | 44 +++--- federationapi/consumers/presence.go | 10 +- federationapi/consumers/receipts.go | 34 ++--- federationapi/consumers/sendtodevice.go | 36 ++--- federationapi/consumers/typing.go | 32 ++--- federationapi/federationapi.go | 4 +- federationapi/federationapi_keys_test.go | 2 +- federationapi/federationapi_test.go | 1 + federationapi/internal/federationclient.go | 2 +- federationapi/internal/keys.go | 2 +- federationapi/internal/perform.go | 2 +- federationapi/producers/syncapi.go | 5 +- federationapi/queue/destinationqueue.go | 64 +++++---- federationapi/queue/queue.go | 30 +++- federationapi/queue/queue_test.go | 2 +- federationapi/routing/publicrooms.go | 25 +++- federationapi/routing/routing.go | 50 +++---- federationapi/storage/postgres/storage.go | 4 +- federationapi/storage/shared/storage.go | 4 +- federationapi/storage/sqlite3/storage.go | 4 +- federationapi/storage/storage.go | 6 +- federationapi/storage/storage_test.go | 2 +- go.mod | 28 ++-- go.sum | 124 ++++++----------- keyserver/internal/internal.go | 63 ++++----- roomserver/api/api.go | 1 + roomserver/api/perform.go | 6 +- roomserver/api/query.go | 7 +- roomserver/internal/perform/perform_admin.go | 11 +- roomserver/internal/perform/perform_invite.go | 4 +- roomserver/internal/perform/perform_join.go | 23 ++-- roomserver/internal/perform/perform_leave.go | 6 +- roomserver/internal/perform/perform_peek.go | 6 +- .../internal/perform/perform_publish.go | 2 +- roomserver/internal/perform/perform_unpeek.go | 2 +- .../internal/perform/perform_upgrade.go | 37 +++-- roomserver/internal/query/query.go | 2 +- roomserver/storage/interface.go | 4 +- .../20221027084407_published_appservice.go | 45 ++++++ .../storage/postgres/published_table.go | 55 ++++++-- roomserver/storage/shared/storage.go | 8 +- .../20221027084407_published_appservice.go | 64 +++++++++ roomserver/storage/sqlite3/published_table.go | 55 ++++++-- roomserver/storage/tables/interface.go | 4 +- .../storage/tables/published_table_test.go | 33 ++++- setup/config/config_clientapi.go | 18 +++ setup/config/config_global.go | 15 ++ setup/config/config_userapi.go | 4 + setup/mscs/msc2836/msc2836.go | 2 +- setup/mscs/msc2946/msc2946.go | 2 +- syncapi/routing/memberships.go | 21 +-- syncapi/streams/stream_pdu.go | 15 +- sytest-whitelist | 4 +- test/testrig/base.go | 2 + userapi/api/api.go | 31 ++++- userapi/internal/api.go | 100 +++++++++++--- userapi/internal/api_logintoken.go | 8 +- userapi/userapi.go | 3 +- userapi/userapi_test.go | 4 +- 84 files changed, 1230 insertions(+), 550 deletions(-) create mode 100644 .github/workflows/schedules.yaml create mode 100644 build/docker/Dockerfile.demo-yggdrasil create mode 100644 roomserver/storage/postgres/deltas/20221027084407_published_appservice.go create mode 100644 roomserver/storage/sqlite3/deltas/20221027084407_published_appservice.go diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml new file mode 100644 index 000000000..c07917248 --- /dev/null +++ b/.github/workflows/schedules.yaml @@ -0,0 +1,128 @@ +name: Scheduled + +on: + schedule: + - cron: '0 0 * * *' # every day at midnight + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # run go test with different go versions + test: + timeout-minutes: 20 + name: Unit tests (Go ${{ matrix.go }}) + runs-on: ubuntu-latest + # Service containers to run with `container-job` + services: + # Label used to access the service container + postgres: + # Docker Hub image + image: postgres:13-alpine + # Provide the password for postgres + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: dendrite + ports: + # Maps tcp port 5432 on service container to the host + - 5432:5432 + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + strategy: + fail-fast: false + matrix: + go: ["1.18", "1.19"] + steps: + - uses: actions/checkout@v3 + - name: Setup go + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go }} + - uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go${{ matrix.go }}-test-race-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go${{ matrix.go }}-test-race- + - run: go test -race ./... + env: + POSTGRES_HOST: localhost + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: dendrite + + # Dummy step to gate other tests on without repeating the whole list + initial-tests-done: + name: Initial tests passed + needs: [test] + runs-on: ubuntu-latest + if: ${{ !cancelled() }} # Run this even if prior jobs were skipped + steps: + - name: Check initial tests passed + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} + + # run Sytest in different variations + sytest: + timeout-minutes: 60 + needs: initial-tests-done + name: "Sytest (${{ matrix.label }})" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - label: SQLite + + - label: SQLite, full HTTP APIs + api: full-http + + - label: PostgreSQL + postgres: postgres + + - label: PostgreSQL, full HTTP APIs + postgres: postgres + api: full-http + container: + image: matrixdotorg/sytest-dendrite:latest + volumes: + - ${{ github.workspace }}:/src + env: + POSTGRES: ${{ matrix.postgres && 1}} + API: ${{ matrix.api && 1 }} + SYTEST_BRANCH: ${{ github.head_ref }} + RACE_DETECTION: 1 + steps: + - uses: actions/checkout@v2 + - name: Run Sytest + run: /bootstrap.sh dendrite + working-directory: /src + - name: Summarise results.tap + if: ${{ always() }} + run: /sytest/scripts/tap_to_gha.pl /logs/results.tap + - name: Sytest List Maintenance + if: ${{ always() }} + run: /src/show-expected-fail-tests.sh /logs/results.tap /src/sytest-whitelist /src/sytest-blacklist + continue-on-error: true # not fatal + - name: Are We Synapse Yet? + if: ${{ always() }} + run: /src/are-we-synapse-yet.py /logs/results.tap -v + continue-on-error: true # not fatal + - name: Upload Sytest logs + uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }}) + path: | + /logs/results.tap + /logs/**/*.log* diff --git a/build/docker/Dockerfile.demo-yggdrasil b/build/docker/Dockerfile.demo-yggdrasil new file mode 100644 index 000000000..76bf35823 --- /dev/null +++ b/build/docker/Dockerfile.demo-yggdrasil @@ -0,0 +1,25 @@ +FROM docker.io/golang:1.19-alpine AS base + +RUN apk --update --no-cache add bash build-base + +WORKDIR /build + +COPY . /build + +RUN mkdir -p bin +RUN go build -trimpath -o bin/ ./cmd/dendrite-demo-yggdrasil +RUN go build -trimpath -o bin/ ./cmd/create-account +RUN go build -trimpath -o bin/ ./cmd/generate-keys + +FROM alpine:latest +LABEL org.opencontainers.image.title="Dendrite (Yggdrasil demo)" +LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" +LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" +LABEL org.opencontainers.image.licenses="Apache-2.0" + +COPY --from=base /build/bin/* /usr/bin/ + +VOLUME /etc/dendrite +WORKDIR /etc/dendrite + +ENTRYPOINT ["/usr/bin/dendrite-demo-yggdrasil"] diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 4a96e4bef..adb4e40a6 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -101,18 +101,46 @@ func (m *DendriteMonolith) SessionCount() int { return len(m.PineconeQUIC.Protocol("matrix").Sessions()) } -func (m *DendriteMonolith) RegisterNetworkInterface(name string, index int, mtu int, up bool, broadcast bool, loopback bool, pointToPoint bool, multicast bool, addrs string) { - m.PineconeMulticast.RegisterInterface(pineconeMulticast.InterfaceInfo{ - Name: name, - Index: index, - Mtu: mtu, - Up: up, - Broadcast: broadcast, - Loopback: loopback, - PointToPoint: pointToPoint, - Multicast: multicast, - Addrs: addrs, - }) +type InterfaceInfo struct { + Name string + Index int + Mtu int + Up bool + Broadcast bool + Loopback bool + PointToPoint bool + Multicast bool + Addrs string +} + +type InterfaceRetriever interface { + CacheCurrentInterfaces() int + GetCachedInterface(index int) *InterfaceInfo +} + +func (m *DendriteMonolith) RegisterNetworkCallback(intfCallback InterfaceRetriever) { + callback := func() []pineconeMulticast.InterfaceInfo { + count := intfCallback.CacheCurrentInterfaces() + intfs := []pineconeMulticast.InterfaceInfo{} + for i := 0; i < count; i++ { + iface := intfCallback.GetCachedInterface(i) + if iface != nil { + intfs = append(intfs, pineconeMulticast.InterfaceInfo{ + Name: iface.Name, + Index: iface.Index, + Mtu: iface.Mtu, + Up: iface.Up, + Broadcast: iface.Broadcast, + Loopback: iface.Loopback, + PointToPoint: iface.PointToPoint, + Multicast: iface.Multicast, + Addrs: iface.Addrs, + }) + } + } + return intfs + } + m.PineconeMulticast.RegisterNetworkCallback(callback) } func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) { diff --git a/clientapi/auth/login_publickey_ethereum.go b/clientapi/auth/login_publickey_ethereum.go index 1ce41d886..33c0a16d4 100644 --- a/clientapi/auth/login_publickey_ethereum.go +++ b/clientapi/auth/login_publickey_ethereum.go @@ -67,7 +67,7 @@ func (pk LoginPublicKeyEthereum) GetType() string { } func (pk LoginPublicKeyEthereum) AccountExists(ctx context.Context) (string, *jsonerror.MatrixError) { - localPart, err := userutil.ParseUsernameParam(pk.UserId, &pk.config.Matrix.ServerName) + localPart, _, err := userutil.ParseUsernameParam(pk.UserId, pk.config.Matrix) if err != nil { // userId does not exist return "", jsonerror.Forbidden("the address is incorrect, or the account does not exist.") diff --git a/clientapi/auth/password.go b/clientapi/auth/password.go index 3bd77eb3d..1c8540e41 100644 --- a/clientapi/auth/password.go +++ b/clientapi/auth/password.go @@ -74,7 +74,7 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, JSON: jsonerror.BadJSON("A password must be supplied."), } } - localpart, err := userutil.ParseUsernameParam(username, &t.Config.Matrix.ServerName) + localpart, _, err := userutil.ParseUsernameParam(username, t.Config.Matrix) if err != nil { return nil, &util.JSONResponse{ Code: http.StatusUnauthorized, diff --git a/clientapi/routing/admin.go b/clientapi/routing/admin.go index 89c269f1a..69bca13be 100644 --- a/clientapi/routing/admin.go +++ b/clientapi/routing/admin.go @@ -70,7 +70,7 @@ func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, device *userapi if err != nil { return util.MessageResponse(http.StatusBadRequest, err.Error()) } - if domain != cfg.Matrix.ServerName { + if !cfg.Matrix.IsLocalServerName(domain) { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.MissingArgument("User ID must belong to this server."), @@ -169,7 +169,7 @@ func AdminMarkAsStale(req *http.Request, cfg *config.ClientAPI, keyAPI api.Clien if err != nil { return util.MessageResponse(http.StatusBadRequest, err.Error()) } - if domain == cfg.Matrix.ServerName { + if cfg.Matrix.IsLocalServerName(domain) { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.InvalidParam("Can not mark local device list as stale"), diff --git a/clientapi/routing/auth_fallback.go b/clientapi/routing/auth_fallback.go index abfe830fb..ad870993e 100644 --- a/clientapi/routing/auth_fallback.go +++ b/clientapi/routing/auth_fallback.go @@ -31,8 +31,7 @@ const recaptchaTemplate = ` Authentication - +