diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index d6c54ead1..bbbdd82a4 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,7 +2,7 @@ -* [ ] I have added tests for PR _or_ I have justified why this PR doesn't need tests. +* [ ] I have added Go unit tests or [Complement integration tests](https://github.com/matrix-org/complement) for this PR _or_ I have justified why this PR doesn't need tests * [ ] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) _or_ I have already signed off privately Signed-off-by: `Your Name ` diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index a8271b675..fa4282384 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -26,22 +26,14 @@ jobs: uses: actions/setup-go@v3 with: go-version: 1.18 - - - uses: actions/cache@v2 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-wasm-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-wasm + cache: true - name: Install Node uses: actions/setup-node@v2 with: node-version: 14 - - uses: actions/cache@v2 + - uses: actions/cache@v3 with: path: ~/.npm key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} @@ -109,19 +101,12 @@ jobs: uses: actions/setup-go@v3 with: go-version: ${{ matrix.go }} + cache: true - name: Set up gotestfmt uses: gotesttools/gotestfmt-action@v2 with: # Optional: pass GITHUB_TOKEN to avoid rate limiting. token: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/cache@v3 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go${{ matrix.go }}-test-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go${{ matrix.go }}-test- - run: go test -json -v ./... 2>&1 | gotestfmt env: POSTGRES_HOST: localhost @@ -146,17 +131,17 @@ jobs: uses: actions/setup-go@v3 with: go-version: ${{ matrix.go }} - - name: Install dependencies x86 - if: ${{ matrix.goarch == '386' }} - run: sudo apt update && sudo apt-get install -y gcc-multilib - uses: actions/cache@v3 with: path: | ~/.cache/go-build ~/go/pkg/mod - key: ${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }} + key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }} restore-keys: | - ${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goarch }}- + key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}- + - name: Install dependencies x86 + if: ${{ matrix.goarch == '386' }} + run: sudo apt update && sudo apt-get install -y gcc-multilib - env: GOOS: ${{ matrix.goos }} GOARCH: ${{ matrix.goarch }} @@ -180,16 +165,16 @@ jobs: uses: actions/setup-go@v3 with: go-version: ${{ matrix.go }} - - name: Install dependencies - run: sudo apt update && sudo apt install -y gcc-mingw-w64-x86-64 # install required gcc - uses: actions/cache@v3 with: path: | ~/.cache/go-build ~/go/pkg/mod - key: ${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goos }}-${{ hashFiles('**/go.sum') }} + key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }} restore-keys: | - ${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goos }} + key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}- + - name: Install dependencies + run: sudo apt update && sudo apt install -y gcc-mingw-w64-x86-64 # install required gcc - env: GOOS: ${{ matrix.goos }} GOARCH: ${{ matrix.goarch }} @@ -221,18 +206,13 @@ jobs: uses: actions/setup-go@v3 with: go-version: "1.18" - - uses: actions/cache@v3 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-upgrade-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-upgrade + cache: true - name: Build upgrade-tests run: go build ./cmd/dendrite-upgrade-tests - - name: Test upgrade + - name: Test upgrade (PostgreSQL) run: ./dendrite-upgrade-tests --head . + - name: Test upgrade (SQLite) + run: ./dendrite-upgrade-tests --sqlite --head . # run database upgrade tests, skipping over one version upgrade_test_direct: @@ -246,17 +226,12 @@ jobs: uses: actions/setup-go@v3 with: go-version: "1.18" - - uses: actions/cache@v3 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-upgrade-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-upgrade + cache: true - name: Build upgrade-tests run: go build ./cmd/dendrite-upgrade-tests - - name: Test upgrade + - name: Test upgrade (PostgreSQL) + run: ./dendrite-upgrade-tests -direct -from HEAD-2 --head . + - name: Test upgrade (SQLite) run: ./dendrite-upgrade-tests -direct -from HEAD-2 --head . # run Sytest in different variations @@ -269,11 +244,18 @@ jobs: fail-fast: false matrix: include: - - label: SQLite + - label: SQLite native - - label: SQLite, full HTTP APIs + - label: SQLite Cgo + cgo: 1 + + - label: SQLite native, full HTTP APIs api: full-http + - label: SQLite Cgo, full HTTP APIs + api: full-http + cgo: 1 + - label: PostgreSQL postgres: postgres @@ -284,12 +266,23 @@ jobs: image: matrixdotorg/sytest-dendrite:latest volumes: - ${{ github.workspace }}:/src + - /root/.cache/go-build:/github/home/.cache/go-build + - /root/.cache/go-mod:/gopath/pkg/mod env: POSTGRES: ${{ matrix.postgres && 1}} API: ${{ matrix.api && 1 }} SYTEST_BRANCH: ${{ github.head_ref }} + CGO_ENABLED: ${{ matrix.cgo && 1 }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + /gopath/pkg/mod + key: ${{ runner.os }}-go-sytest-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-sytest- - name: Run Sytest run: /bootstrap.sh dendrite working-directory: /src @@ -323,17 +316,28 @@ jobs: fail-fast: false matrix: include: - - label: SQLite + - label: SQLite native + cgo: 0 - - label: SQLite, full HTTP APIs + - label: SQLite Cgo + cgo: 1 + + - label: SQLite native, full HTTP APIs api: full-http + cgo: 0 + + - label: SQLite Cgo, full HTTP APIs + api: full-http + cgo: 1 - label: PostgreSQL postgres: Postgres + cgo: 0 - label: PostgreSQL, full HTTP APIs postgres: Postgres api: full-http + cgo: 0 steps: # Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env to run Complement. # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path @@ -341,16 +345,14 @@ jobs: run: | echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH echo "~/go/bin" >> $GITHUB_PATH - - name: "Install Complement Dependencies" # We don't need to install Go because it is included on the Ubuntu 20.04 image: # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64 run: | sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest - - - name: Run actions/checkout@v2 for dendrite - uses: actions/checkout@v2 + - name: Run actions/checkout@v3 for dendrite + uses: actions/checkout@v3 with: path: dendrite @@ -374,12 +376,10 @@ jobs: if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then continue fi - (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break done - # Build initial Dendrite image - - run: docker build -t complement-dendrite -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile . + - run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile . working-directory: dendrite env: DOCKER_BUILDKIT: 1 @@ -391,7 +391,7 @@ jobs: shell: bash name: Run Complement Tests env: - COMPLEMENT_BASE_IMAGE: complement-dendrite:latest + COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }} API: ${{ matrix.api && 1 }} working-directory: complement @@ -418,6 +418,7 @@ jobs: permissions: packages: write contents: read + security-events: write # To upload Trivy sarif files if: github.repository == 'matrix-org/dendrite' && github.ref_name == 'main' needs: [integration-tests-done] uses: matrix-org/dendrite/.github/workflows/docker.yml@main diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b4e24e52f..846844173 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -24,23 +24,29 @@ jobs: permissions: contents: read packages: write + security-events: write # To upload Trivy sarif files steps: - name: Checkout - uses: actions/checkout@v2 - - name: Get release tag + uses: actions/checkout@v3 + - name: Get release tag & build flags if: github.event_name == 'release' # Only for GitHub releases - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + run: | + echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV + BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) + [ ${BRANCH} == "main" ] && BRANCH="" + echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ env.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_TOKEN }} - name: Login to GitHub Containers - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -49,27 +55,41 @@ jobs: - name: Build main monolith image if: github.ref_name == 'main' id: docker_build_monolith - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.monolith + build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} + target: monolith platforms: ${{ env.PLATFORMS }} push: true tags: | ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} + format: "sarif" + output: "trivy-results.sarif" + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" + - name: Build release monolith image if: github.event_name == 'release' # Only for GitHub releases id: docker_build_monolith_release - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.monolith + build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} + target: monolith platforms: ${{ env.PLATFORMS }} push: true tags: | @@ -84,23 +104,29 @@ jobs: permissions: contents: read packages: write + security-events: write # To upload Trivy sarif files steps: - name: Checkout - uses: actions/checkout@v2 - - name: Get release tag + uses: actions/checkout@v3 + - name: Get release tag & build flags if: github.event_name == 'release' # Only for GitHub releases - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + run: | + echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV + BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) + [ ${BRANCH} == "main" ] && BRANCH="" + echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ env.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_TOKEN }} - name: Login to GitHub Containers - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -109,27 +135,41 @@ jobs: - name: Build main polylith image if: github.ref_name == 'main' id: docker_build_polylith - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.polylith + build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} + target: polylith platforms: ${{ env.PLATFORMS }} push: true tags: | ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} + format: "sarif" + output: "trivy-results.sarif" + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" + - name: Build release polylith image if: github.event_name == 'release' # Only for GitHub releases id: docker_build_polylith_release - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.polylith + build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} + target: polylith platforms: ${{ env.PLATFORMS }} push: true tags: | @@ -146,34 +186,40 @@ jobs: packages: write steps: - name: Checkout - uses: actions/checkout@v2 - - name: Get release tag + uses: actions/checkout@v3 + - name: Get release tag & build flags if: github.event_name == 'release' # Only for GitHub releases - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + run: | + echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV + BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) + [ ${BRANCH} == "main" ] && BRANCH="" + echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ env.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_TOKEN }} - name: Login to GitHub Containers - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build main pinecone demo image + - name: Build main Pinecone demo image if: github.ref_name == 'main' id: docker_build_demo_pinecone - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . + build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} file: ./build/docker/Dockerfile.demo-pinecone platforms: ${{ env.PLATFORMS }} push: true @@ -181,19 +227,87 @@ jobs: ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }} ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }} - - name: Build release pinecone demo image + - name: Build release Pinecone demo image if: github.event_name == 'release' # Only for GitHub releases id: docker_build_demo_pinecone_release - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . + build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} file: ./build/docker/Dockerfile.demo-pinecone platforms: ${{ env.PLATFORMS }} push: true tags: | - ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:latest - ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:${{ env.RELEASE_VERSION }} - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:latest - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:${{ env.RELEASE_VERSION }} + ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:latest + ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }} + ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:latest + ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }} + + demo-yggdrasil: + name: Yggdrasil demo image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Get release tag & build flags + if: github.event_name == 'release' # Only for GitHub releases + run: | + echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV + BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) + [ ${BRANCH} == "main" ] && BRANCH="" + echo "BRANCH=${BRANCH}" >> $GITHUB_ENV + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ env.DOCKER_HUB_USER }} + password: ${{ secrets.DOCKER_TOKEN }} + - name: Login to GitHub Containers + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build main Yggdrasil demo image + if: github.ref_name == 'main' + id: docker_build_demo_yggdrasil + uses: docker/build-push-action@v3 + with: + cache-from: type=gha + cache-to: type=gha,mode=max + context: . + build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} + file: ./build/docker/Dockerfile.demo-yggdrasil + platforms: ${{ env.PLATFORMS }} + push: true + tags: | + ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ github.ref_name }} + ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ github.ref_name }} + + - name: Build release Yggdrasil demo image + if: github.event_name == 'release' # Only for GitHub releases + id: docker_build_demo_yggdrasil_release + uses: docker/build-push-action@v3 + with: + cache-from: type=gha + cache-to: type=gha,mode=max + context: . + build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} + file: ./build/docker/Dockerfile.demo-yggdrasil + platforms: ${{ env.PLATFORMS }} + push: true + tags: | + ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:latest + ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }} + ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:latest + ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }} diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml new file mode 100644 index 000000000..ff4d47187 --- /dev/null +++ b/.github/workflows/schedules.yaml @@ -0,0 +1,135 @@ +name: Scheduled + +on: + schedule: + - cron: '0 0 * * *' # every day at midnight + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # run go test with different go versions + test: + timeout-minutes: 20 + name: Unit tests (Go ${{ matrix.go }}) + runs-on: ubuntu-latest + # Service containers to run with `container-job` + services: + # Label used to access the service container + postgres: + # Docker Hub image + image: postgres:13-alpine + # Provide the password for postgres + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: dendrite + ports: + # Maps tcp port 5432 on service container to the host + - 5432:5432 + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + strategy: + fail-fast: false + matrix: + go: ["1.18", "1.19"] + steps: + - uses: actions/checkout@v3 + - name: Setup go + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go }} + - name: Set up gotestfmt + uses: gotesttools/gotestfmt-action@v2 + with: + # Optional: pass GITHUB_TOKEN to avoid rate limiting. + token: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go${{ matrix.go }}-test-race-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go${{ matrix.go }}-test-race- + - run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt + env: + POSTGRES_HOST: localhost + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: dendrite + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + + # Dummy step to gate other tests on without repeating the whole list + initial-tests-done: + name: Initial tests passed + needs: [test] + runs-on: ubuntu-latest + if: ${{ !cancelled() }} # Run this even if prior jobs were skipped + steps: + - name: Check initial tests passed + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} + + # run Sytest in different variations + sytest: + timeout-minutes: 60 + needs: initial-tests-done + name: "Sytest (${{ matrix.label }})" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - label: SQLite + + - label: SQLite, full HTTP APIs + api: full-http + + - label: PostgreSQL + postgres: postgres + + - label: PostgreSQL, full HTTP APIs + postgres: postgres + api: full-http + container: + image: matrixdotorg/sytest-dendrite:latest + volumes: + - ${{ github.workspace }}:/src + env: + POSTGRES: ${{ matrix.postgres && 1}} + API: ${{ matrix.api && 1 }} + SYTEST_BRANCH: ${{ github.head_ref }} + RACE_DETECTION: 1 + steps: + - uses: actions/checkout@v2 + - name: Run Sytest + run: /bootstrap.sh dendrite + working-directory: /src + - name: Summarise results.tap + if: ${{ always() }} + run: /sytest/scripts/tap_to_gha.pl /logs/results.tap + - name: Sytest List Maintenance + if: ${{ always() }} + run: /src/show-expected-fail-tests.sh /logs/results.tap /src/sytest-whitelist /src/sytest-blacklist + continue-on-error: true # not fatal + - name: Are We Synapse Yet? + if: ${{ always() }} + run: /src/are-we-synapse-yet.py /logs/results.tap -v + continue-on-error: true # not fatal + - name: Upload Sytest logs + uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }}) + path: | + /logs/results.tap + /logs/**/*.log* diff --git a/CHANGES.md b/CHANGES.md index 1ed87824a..cdeb1dea3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,56 @@ # Changelog +## Dendrite 0.10.7 (2022-11-04) + +### Features + +* Dendrite will now use a native SQLite port when building with `CGO_ENABLED=0` +* A number of `thirdparty` endpoints have been added, improving support for appservices + +### Fixes + +* The `"state"` section of the `/sync` response is no longer limited, so state events should not be dropped unexpectedly +* The deduplication of the `"timeline"` and `"state"` sections in `/sync` is now performed after applying history visibility, so state events should not be dropped unexpectedly +* The `prev_batch` token returned by `/sync` is now calculated after applying history visibility, so that the pagination boundaries are correct +* The room summary membership counts in `/sync` should now be calculated properly in more cases +* A false membership leave event should no longer be sent down `/sync` as a result of retiring an accepted invite (contributed by [tak-hntlabs](https://github.com/tak-hntlabs)) +* Presence updates are now only sent to other servers for which the user shares rooms +* A bug which could cause a panic when converting events into the `ClientEvent` format has been fixed + +## Dendrite 0.10.6 (2022-11-01) + +### Features + +* History visibility checks have been optimised, which should speed up response times on a variety of endpoints (including `/sync`, `/messages`, `/context` and others) and reduce database load +* The built-in NATS Server has been updated to version 2.9.4 +* Some other minor dependencies have been updated + +### Fixes + +* A panic has been fixed in the sync API PDU stream which could cause requests to fail +* The `/members` response now contains the `room_id` field, which may fix some E2EE problems with clients using the JS SDK (contributed by [ashkitten](https://github.com/ashkitten)) +* The auth difference calculation in state resolution v2 has been tweaked for clarity (and moved into gomatrixserverlib with the rest of the state resolution code) + +## Dendrite 0.10.5 (2022-10-31) + +### Features + +* It is now possible to use hCaptcha instead of reCAPTCHA for protecting registration +* A new `auto_join_rooms` configuration option has been added for automatically joining new users to a set of rooms +* A new `/_dendrite/admin/downloadState/{serverName}/{roomID}` endpoint has been added, which allows a server administrator to attempt to repair a room with broken room state by downloading a state snapshot from another federated server in the room + +### Fixes + +* Querying cross-signing keys for users should now be considerably faster +* A bug in state resolution where some events were not correctly selected for third-party invites has been fixed +* A bug in state resolution which could result in `not in room` event rejections has been fixed +* When accepting a DM invite, it should now be possible to see messages that were sent before the invite was accepted +* Claiming remote E2EE one-time keys has been refactored and should be more reliable now +* Various fixes have been made to the `/members` endpoint, which may help with E2EE reliability and clients rendering memberships +* A race condition in the federation API destination queues has been fixed when associating queued events with remote server destinations +* A bug in the sync API where too many events were selected resulting in high CPU usage has been fixed +* Configuring the avatar URL for the Server Notices user should work correctly now + ## Dendrite 0.10.4 (2022-10-21) ### Features diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..a9bbce925 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,92 @@ +#syntax=docker/dockerfile:1.2 + +# +# base installs required dependencies and runs go mod download to cache dependencies +# +FROM --platform=${BUILDPLATFORM} docker.io/golang:1.19-alpine AS base +RUN apk --update --no-cache add bash build-base curl + +# +# build creates all needed binaries +# +FROM --platform=${BUILDPLATFORM} base AS build +WORKDIR /src +ARG TARGETOS +ARG TARGETARCH +ARG FLAGS +RUN --mount=target=. \ + --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + USERARCH=`go env GOARCH` \ + GOARCH="$TARGETARCH" \ + GOOS="linux" \ + CGO_ENABLED=$([ "$TARGETARCH" = "$USERARCH" ] && echo "1" || echo "0") \ + go build -v -ldflags="${FLAGS}" -trimpath -o /out/ ./cmd/... + +# +# The dendrite base image +# +FROM alpine:latest AS dendrite-base +LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" +LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" +LABEL org.opencontainers.image.licenses="Apache-2.0" +LABEL org.opencontainers.image.documentation="https://matrix-org.github.io/dendrite/" +LABEL org.opencontainers.image.vendor="The Matrix.org Foundation C.I.C." + +# +# Builds the polylith image and only contains the polylith binary +# +FROM dendrite-base AS polylith +LABEL org.opencontainers.image.title="Dendrite (Polylith)" + +COPY --from=build /out/dendrite-polylith-multi /usr/bin/ + +VOLUME /etc/dendrite +WORKDIR /etc/dendrite + +ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"] + +# +# Builds the monolith image and contains all required binaries +# +FROM dendrite-base AS monolith +LABEL org.opencontainers.image.title="Dendrite (Monolith)" + +COPY --from=build /out/create-account /usr/bin/create-account +COPY --from=build /out/generate-config /usr/bin/generate-config +COPY --from=build /out/generate-keys /usr/bin/generate-keys +COPY --from=build /out/dendrite-monolith-server /usr/bin/dendrite-monolith-server + +VOLUME /etc/dendrite +WORKDIR /etc/dendrite + +ENTRYPOINT ["/usr/bin/dendrite-monolith-server"] +EXPOSE 8008 8448 + +# +# Builds the Complement image, used for integration tests +# +FROM base AS complement +LABEL org.opencontainers.image.title="Dendrite (Complement)" +RUN apk add --no-cache sqlite openssl ca-certificates + +COPY --from=build /out/generate-config /usr/bin/generate-config +COPY --from=build /out/generate-keys /usr/bin/generate-keys +COPY --from=build /out/dendrite-monolith-server /usr/bin/dendrite-monolith-server + +WORKDIR /dendrite +RUN /usr/bin/generate-keys --private-key matrix_key.pem && \ + mkdir /ca && \ + openssl genrsa -out /ca/ca.key 2048 && \ + openssl req -new -x509 -key /ca/ca.key -days 3650 -subj "/C=GB/ST=London/O=matrix.org/CN=Complement CA" -out /ca/ca.crt + +ENV SERVER_NAME=localhost +ENV API=0 +EXPOSE 8008 8448 + +# At runtime, generate TLS cert based on the CA now mounted at /ca +# At runtime, replace the SERVER_NAME with what we are told +CMD /usr/bin/generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /ca/ca.crt --tls-authority-key /ca/ca.key && \ + /usr/bin/generate-config -server $SERVER_NAME --ci > dendrite.yaml && \ + cp /ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \ + /usr/bin/dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0} diff --git a/appservice/api/query.go b/appservice/api/query.go index 4d1cf9474..eb567b2ee 100644 --- a/appservice/api/query.go +++ b/appservice/api/query.go @@ -19,11 +19,13 @@ package api import ( "context" + "encoding/json" "errors" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" ) // AppServiceInternalAPI is used to query user and room alias data from application @@ -41,6 +43,10 @@ type AppServiceInternalAPI interface { req *UserIDExistsRequest, resp *UserIDExistsResponse, ) error + + Locations(ctx context.Context, req *LocationRequest, resp *LocationResponse) error + User(ctx context.Context, request *UserRequest, response *UserResponse) error + Protocols(ctx context.Context, req *ProtocolRequest, resp *ProtocolResponse) error } // RoomAliasExistsRequest is a request to an application service @@ -77,6 +83,73 @@ type UserIDExistsResponse struct { UserIDExists bool `json:"exists"` } +const ( + ASProtocolPath = "/_matrix/app/unstable/thirdparty/protocol/" + ASUserPath = "/_matrix/app/unstable/thirdparty/user" + ASLocationPath = "/_matrix/app/unstable/thirdparty/location" +) + +type ProtocolRequest struct { + Protocol string `json:"protocol,omitempty"` +} + +type ProtocolResponse struct { + Protocols map[string]ASProtocolResponse `json:"protocols"` + Exists bool `json:"exists"` +} + +type ASProtocolResponse struct { + FieldTypes map[string]FieldType `json:"field_types,omitempty"` // NOTSPEC: field_types is required by the spec + Icon string `json:"icon"` + Instances []ProtocolInstance `json:"instances"` + LocationFields []string `json:"location_fields"` + UserFields []string `json:"user_fields"` +} + +type FieldType struct { + Placeholder string `json:"placeholder"` + Regexp string `json:"regexp"` +} + +type ProtocolInstance struct { + Description string `json:"desc"` + Icon string `json:"icon,omitempty"` + NetworkID string `json:"network_id,omitempty"` // NOTSPEC: network_id is required by the spec + Fields json.RawMessage `json:"fields,omitempty"` // NOTSPEC: fields is required by the spec +} + +type UserRequest struct { + Protocol string `json:"protocol"` + Params string `json:"params"` +} + +type UserResponse struct { + Users []ASUserResponse `json:"users,omitempty"` + Exists bool `json:"exists,omitempty"` +} + +type ASUserResponse struct { + Protocol string `json:"protocol"` + UserID string `json:"userid"` + Fields json.RawMessage `json:"fields"` +} + +type LocationRequest struct { + Protocol string `json:"protocol"` + Params string `json:"params"` +} + +type LocationResponse struct { + Locations []ASLocationResponse `json:"locations,omitempty"` + Exists bool `json:"exists,omitempty"` +} + +type ASLocationResponse struct { + Alias string `json:"alias"` + Protocol string `json:"protocol"` + Fields json.RawMessage `json:"fields"` +} + // RetrieveUserProfile is a wrapper that queries both the local database and // application services for a given user's profile // TODO: Remove this, it's called from federationapi and clientapi but is a pure function diff --git a/appservice/appservice.go b/appservice/appservice.go index 9000adb1d..b3c28dbde 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -18,6 +18,7 @@ import ( "context" "crypto/tls" "net/http" + "sync" "time" "github.com/gorilla/mux" @@ -31,6 +32,7 @@ import ( "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" ) // AddInternalRoutes registers HTTP handlers for internal API calls @@ -58,8 +60,10 @@ func NewInternalAPI( // Create appserivce query API with an HTTP client that will be used for all // outbound and inbound requests (inbound only for the internal API) appserviceQueryAPI := &query.AppServiceQueryAPI{ - HTTPClient: client, - Cfg: &base.Cfg.AppServiceAPI, + HTTPClient: client, + Cfg: &base.Cfg.AppServiceAPI, + ProtocolCache: map[string]appserviceAPI.ASProtocolResponse{}, + CacheMu: sync.Mutex{}, } if len(base.Cfg.Derived.ApplicationServices) == 0 { @@ -71,7 +75,7 @@ func NewInternalAPI( // events to be sent out. for _, appservice := range base.Cfg.Derived.ApplicationServices { // Create bot account for this AS if it doesn't already exist - if err := generateAppServiceAccount(userAPI, appservice); err != nil { + if err := generateAppServiceAccount(userAPI, appservice, base.Cfg.Global.ServerName); err != nil { logrus.WithFields(logrus.Fields{ "appservice": appservice.ID, }).WithError(err).Panicf("failed to generate bot account for appservice") @@ -98,11 +102,13 @@ func NewInternalAPI( func generateAppServiceAccount( userAPI userapi.AppserviceUserAPI, as config.ApplicationService, + serverName gomatrixserverlib.ServerName, ) error { var accRes userapi.PerformAccountCreationResponse err := userAPI.PerformAccountCreation(context.Background(), &userapi.PerformAccountCreationRequest{ AccountType: userapi.AccountTypeAppService, Localpart: as.SenderLocalpart, + ServerName: serverName, AppServiceID: as.ID, OnConflict: userapi.ConflictUpdate, }, &accRes) @@ -112,6 +118,7 @@ func generateAppServiceAccount( var devRes userapi.PerformDeviceCreationResponse err = userAPI.PerformDeviceCreation(context.Background(), &userapi.PerformDeviceCreationRequest{ Localpart: as.SenderLocalpart, + ServerName: serverName, AccessToken: as.ASToken, DeviceID: &as.SenderLocalpart, DeviceDisplayName: &as.SenderLocalpart, diff --git a/appservice/inthttp/client.go b/appservice/inthttp/client.go index 3ae2c9278..f7f164877 100644 --- a/appservice/inthttp/client.go +++ b/appservice/inthttp/client.go @@ -13,6 +13,9 @@ import ( const ( AppServiceRoomAliasExistsPath = "/appservice/RoomAliasExists" AppServiceUserIDExistsPath = "/appservice/UserIDExists" + AppServiceLocationsPath = "/appservice/locations" + AppServiceUserPath = "/appservice/users" + AppServiceProtocolsPath = "/appservice/protocols" ) // httpAppServiceQueryAPI contains the URL to an appservice query API and a @@ -58,3 +61,24 @@ func (h *httpAppServiceQueryAPI) UserIDExists( h.httpClient, ctx, request, response, ) } + +func (h *httpAppServiceQueryAPI) Locations(ctx context.Context, request *api.LocationRequest, response *api.LocationResponse) error { + return httputil.CallInternalRPCAPI( + "ASLocation", h.appserviceURL+AppServiceLocationsPath, + h.httpClient, ctx, request, response, + ) +} + +func (h *httpAppServiceQueryAPI) User(ctx context.Context, request *api.UserRequest, response *api.UserResponse) error { + return httputil.CallInternalRPCAPI( + "ASUser", h.appserviceURL+AppServiceUserPath, + h.httpClient, ctx, request, response, + ) +} + +func (h *httpAppServiceQueryAPI) Protocols(ctx context.Context, request *api.ProtocolRequest, response *api.ProtocolResponse) error { + return httputil.CallInternalRPCAPI( + "ASProtocols", h.appserviceURL+AppServiceProtocolsPath, + h.httpClient, ctx, request, response, + ) +} diff --git a/appservice/inthttp/server.go b/appservice/inthttp/server.go index 01d9f9895..ccf5c83d8 100644 --- a/appservice/inthttp/server.go +++ b/appservice/inthttp/server.go @@ -2,6 +2,7 @@ package inthttp import ( "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/internal/httputil" ) @@ -17,4 +18,19 @@ func AddRoutes(a api.AppServiceInternalAPI, internalAPIMux *mux.Router) { AppServiceUserIDExistsPath, httputil.MakeInternalRPCAPI("AppserviceUserIDExists", a.UserIDExists), ) + + internalAPIMux.Handle( + AppServiceProtocolsPath, + httputil.MakeInternalRPCAPI("AppserviceProtocols", a.Protocols), + ) + + internalAPIMux.Handle( + AppServiceLocationsPath, + httputil.MakeInternalRPCAPI("AppserviceLocations", a.Locations), + ) + + internalAPIMux.Handle( + AppServiceUserPath, + httputil.MakeInternalRPCAPI("AppserviceUser", a.User), + ) } diff --git a/appservice/query/query.go b/appservice/query/query.go index 53b34cb18..2348eab4b 100644 --- a/appservice/query/query.go +++ b/appservice/query/query.go @@ -18,13 +18,18 @@ package query import ( "context" + "encoding/json" + "io" "net/http" "net/url" + "strings" + "sync" + + "github.com/opentracing/opentracing-go" + log "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/setup/config" - opentracing "github.com/opentracing/opentracing-go" - log "github.com/sirupsen/logrus" ) const roomAliasExistsPath = "/rooms/" @@ -32,8 +37,10 @@ const userIDExistsPath = "/users/" // AppServiceQueryAPI is an implementation of api.AppServiceQueryAPI type AppServiceQueryAPI struct { - HTTPClient *http.Client - Cfg *config.AppServiceAPI + HTTPClient *http.Client + Cfg *config.AppServiceAPI + ProtocolCache map[string]api.ASProtocolResponse + CacheMu sync.Mutex } // RoomAliasExists performs a request to '/room/{roomAlias}' on all known @@ -165,3 +172,178 @@ func (a *AppServiceQueryAPI) UserIDExists( response.UserIDExists = false return nil } + +type thirdpartyResponses interface { + api.ASProtocolResponse | []api.ASUserResponse | []api.ASLocationResponse +} + +func requestDo[T thirdpartyResponses](client *http.Client, url string, response *T) (err error) { + origURL := url + // try v1 and unstable appservice endpoints + for _, version := range []string{"v1", "unstable"} { + var resp *http.Response + var body []byte + asURL := strings.Replace(origURL, "unstable", version, 1) + resp, err = client.Get(asURL) + if err != nil { + continue + } + defer resp.Body.Close() // nolint: errcheck + body, err = io.ReadAll(resp.Body) + if err != nil { + continue + } + return json.Unmarshal(body, &response) + } + return err +} + +func (a *AppServiceQueryAPI) Locations( + ctx context.Context, + req *api.LocationRequest, + resp *api.LocationResponse, +) error { + params, err := url.ParseQuery(req.Params) + if err != nil { + return err + } + + for _, as := range a.Cfg.Derived.ApplicationServices { + var asLocations []api.ASLocationResponse + params.Set("access_token", as.HSToken) + + url := as.URL + api.ASLocationPath + if req.Protocol != "" { + url += "/" + req.Protocol + } + + if err := requestDo[[]api.ASLocationResponse](a.HTTPClient, url+"?"+params.Encode(), &asLocations); err != nil { + log.WithError(err).Error("unable to get 'locations' from application service") + continue + } + + resp.Locations = append(resp.Locations, asLocations...) + } + + if len(resp.Locations) == 0 { + resp.Exists = false + return nil + } + resp.Exists = true + return nil +} + +func (a *AppServiceQueryAPI) User( + ctx context.Context, + req *api.UserRequest, + resp *api.UserResponse, +) error { + params, err := url.ParseQuery(req.Params) + if err != nil { + return err + } + + for _, as := range a.Cfg.Derived.ApplicationServices { + var asUsers []api.ASUserResponse + params.Set("access_token", as.HSToken) + + url := as.URL + api.ASUserPath + if req.Protocol != "" { + url += "/" + req.Protocol + } + + if err := requestDo[[]api.ASUserResponse](a.HTTPClient, url+"?"+params.Encode(), &asUsers); err != nil { + log.WithError(err).Error("unable to get 'user' from application service") + continue + } + + resp.Users = append(resp.Users, asUsers...) + } + + if len(resp.Users) == 0 { + resp.Exists = false + return nil + } + resp.Exists = true + return nil +} + +func (a *AppServiceQueryAPI) Protocols( + ctx context.Context, + req *api.ProtocolRequest, + resp *api.ProtocolResponse, +) error { + + // get a single protocol response + if req.Protocol != "" { + + a.CacheMu.Lock() + defer a.CacheMu.Unlock() + if proto, ok := a.ProtocolCache[req.Protocol]; ok { + resp.Exists = true + resp.Protocols = map[string]api.ASProtocolResponse{ + req.Protocol: proto, + } + return nil + } + + response := api.ASProtocolResponse{} + for _, as := range a.Cfg.Derived.ApplicationServices { + var proto api.ASProtocolResponse + if err := requestDo[api.ASProtocolResponse](a.HTTPClient, as.URL+api.ASProtocolPath+req.Protocol, &proto); err != nil { + log.WithError(err).Error("unable to get 'protocol' from application service") + continue + } + + if len(response.Instances) != 0 { + response.Instances = append(response.Instances, proto.Instances...) + } else { + response = proto + } + } + + if len(response.Instances) == 0 { + resp.Exists = false + return nil + } + + resp.Exists = true + resp.Protocols = map[string]api.ASProtocolResponse{ + req.Protocol: response, + } + a.ProtocolCache[req.Protocol] = response + return nil + } + + response := make(map[string]api.ASProtocolResponse, len(a.Cfg.Derived.ApplicationServices)) + + for _, as := range a.Cfg.Derived.ApplicationServices { + for _, p := range as.Protocols { + var proto api.ASProtocolResponse + if err := requestDo[api.ASProtocolResponse](a.HTTPClient, as.URL+api.ASProtocolPath+p, &proto); err != nil { + log.WithError(err).Error("unable to get 'protocol' from application service") + continue + } + existing, ok := response[p] + if !ok { + response[p] = proto + continue + } + existing.Instances = append(existing.Instances, proto.Instances...) + response[p] = existing + } + } + + if len(response) == 0 { + resp.Exists = false + return nil + } + + a.CacheMu.Lock() + defer a.CacheMu.Unlock() + a.ProtocolCache = response + + resp.Exists = true + resp.Protocols = response + return nil +} diff --git a/cmd/dendritejs-pinecone/jsServer.go b/build/dendritejs-pinecone/jsServer.go similarity index 100% rename from cmd/dendritejs-pinecone/jsServer.go rename to build/dendritejs-pinecone/jsServer.go diff --git a/cmd/dendritejs-pinecone/main.go b/build/dendritejs-pinecone/main.go similarity index 100% rename from cmd/dendritejs-pinecone/main.go rename to build/dendritejs-pinecone/main.go diff --git a/cmd/dendritejs-pinecone/main_noop.go b/build/dendritejs-pinecone/main_noop.go similarity index 100% rename from cmd/dendritejs-pinecone/main_noop.go rename to build/dendritejs-pinecone/main_noop.go diff --git a/cmd/dendritejs-pinecone/main_test.go b/build/dendritejs-pinecone/main_test.go similarity index 100% rename from cmd/dendritejs-pinecone/main_test.go rename to build/dendritejs-pinecone/main_test.go diff --git a/build/docker/Dockerfile.demo-pinecone b/build/docker/Dockerfile.demo-pinecone index 133c63c53..facd1e3af 100644 --- a/build/docker/Dockerfile.demo-pinecone +++ b/build/docker/Dockerfile.demo-pinecone @@ -1,5 +1,10 @@ FROM docker.io/golang:1.19-alpine AS base +# +# Needs to be separate from the main Dockerfile for OpenShift, +# as --target is not supported there. +# + RUN apk --update --no-cache add bash build-base WORKDIR /build diff --git a/build/docker/Dockerfile.demo-yggdrasil b/build/docker/Dockerfile.demo-yggdrasil index 76bf35823..efae5496c 100644 --- a/build/docker/Dockerfile.demo-yggdrasil +++ b/build/docker/Dockerfile.demo-yggdrasil @@ -1,5 +1,10 @@ FROM docker.io/golang:1.19-alpine AS base +# +# Needs to be separate from the main Dockerfile for OpenShift, +# as --target is not supported there. +# + RUN apk --update --no-cache add bash build-base WORKDIR /build diff --git a/build/docker/Dockerfile.monolith b/build/docker/Dockerfile.monolith deleted file mode 100644 index 3180e9626..000000000 --- a/build/docker/Dockerfile.monolith +++ /dev/null @@ -1,25 +0,0 @@ -FROM docker.io/golang:1.19-alpine AS base - -RUN apk --update --no-cache add bash build-base - -WORKDIR /build - -COPY . /build - -RUN mkdir -p bin -RUN go build -trimpath -o bin/ ./cmd/dendrite-monolith-server -RUN go build -trimpath -o bin/ ./cmd/create-account -RUN go build -trimpath -o bin/ ./cmd/generate-keys - -FROM alpine:latest -LABEL org.opencontainers.image.title="Dendrite (Monolith)" -LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" -LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" -LABEL org.opencontainers.image.licenses="Apache-2.0" - -COPY --from=base /build/bin/* /usr/bin/ - -VOLUME /etc/dendrite -WORKDIR /etc/dendrite - -ENTRYPOINT ["/usr/bin/dendrite-monolith-server"] diff --git a/build/docker/Dockerfile.polylith b/build/docker/Dockerfile.polylith deleted file mode 100644 index 79f8a5f23..000000000 --- a/build/docker/Dockerfile.polylith +++ /dev/null @@ -1,25 +0,0 @@ -FROM docker.io/golang:1.19-alpine AS base - -RUN apk --update --no-cache add bash build-base - -WORKDIR /build - -COPY . /build - -RUN mkdir -p bin -RUN go build -trimpath -o bin/ ./cmd/dendrite-polylith-multi -RUN go build -trimpath -o bin/ ./cmd/create-account -RUN go build -trimpath -o bin/ ./cmd/generate-keys - -FROM alpine:latest -LABEL org.opencontainers.image.title="Dendrite (Polylith)" -LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" -LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" -LABEL org.opencontainers.image.licenses="Apache-2.0" - -COPY --from=base /build/bin/* /usr/bin/ - -VOLUME /etc/dendrite -WORKDIR /etc/dendrite - -ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"] diff --git a/build/docker/README.md b/build/docker/README.md index 261519fde..7eb20d88f 100644 --- a/build/docker/README.md +++ b/build/docker/README.md @@ -9,15 +9,20 @@ They can be found on Docker Hub: ## Dockerfiles -The `Dockerfile` builds the base image which contains all of the Dendrite -components. The `Dockerfile.component` file takes the given component, as -specified with `--buildarg component=` from the base image and produce -smaller component-specific images, which are substantially smaller and do -not contain the Go toolchain etc. +The `Dockerfile` is a multistage file which can build all four Dendrite +images depending on the supplied `--target`. From the root of the Dendrite +repository, run: + +``` +docker build . --target monolith -t matrixdotorg/dendrite-monolith +docker build . --target polylith -t matrixdotorg/dendrite-monolith +docker build . --target demo-pinecone -t matrixdotorg/dendrite-demo-pinecone +docker build . --target demo-yggdrasil -t matrixdotorg/dendrite-demo-yggdrasil +``` ## Compose files -There are three sample `docker-compose` files: +There are two sample `docker-compose` files: - `docker-compose.monolith.yml` which runs a monolith Dendrite deployment - `docker-compose.polylith.yml` which runs a polylith Dendrite deployment diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh index c2c140685..d97a701ed 100755 --- a/build/docker/images-build.sh +++ b/build/docker/images-build.sh @@ -6,5 +6,7 @@ TAG=${1:-latest} echo "Building tag '${TAG}'" -docker build -t matrixdotorg/dendrite-monolith:${TAG} -f build/docker/Dockerfile.monolith . -docker build -t matrixdotorg/dendrite-polylith:${TAG} -f build/docker/Dockerfile.polylith . \ No newline at end of file +docker build . --target monolith -t matrixdotorg/dendrite-monolith:${TAG} +docker build . --target polylith -t matrixdotorg/dendrite-monolith:${TAG} +docker build . --target demo-pinecone -t matrixdotorg/dendrite-demo-pinecone:${TAG} +docker build . --target demo-yggdrasil -t matrixdotorg/dendrite-demo-yggdrasil:${TAG} \ No newline at end of file diff --git a/build/scripts/Complement.Dockerfile b/build/scripts/Complement.Dockerfile index 14b28498b..79422e645 100644 --- a/build/scripts/Complement.Dockerfile +++ b/build/scripts/Complement.Dockerfile @@ -10,12 +10,13 @@ RUN mkdir /dendrite # Utilise Docker caching when downloading dependencies, this stops us needlessly # downloading dependencies every time. +ARG CGO RUN --mount=target=. \ --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/root/.cache/go-build \ - go build -o /dendrite ./cmd/generate-config && \ - go build -o /dendrite ./cmd/generate-keys && \ - go build -o /dendrite ./cmd/dendrite-monolith-server + CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-config && \ + CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-keys && \ + CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/dendrite-monolith-server WORKDIR /dendrite RUN ./generate-keys --private-key matrix_key.pem diff --git a/build/scripts/ComplementPostgres.Dockerfile b/build/scripts/ComplementPostgres.Dockerfile index 785090b0b..3faf43cc7 100644 --- a/build/scripts/ComplementPostgres.Dockerfile +++ b/build/scripts/ComplementPostgres.Dockerfile @@ -28,12 +28,13 @@ RUN mkdir /dendrite # Utilise Docker caching when downloading dependencies, this stops us needlessly # downloading dependencies every time. +ARG CGO RUN --mount=target=. \ --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/root/.cache/go-build \ - go build -o /dendrite ./cmd/generate-config && \ - go build -o /dendrite ./cmd/generate-keys && \ - go build -o /dendrite ./cmd/dendrite-monolith-server + CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-config && \ + CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-keys && \ + CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/dendrite-monolith-server WORKDIR /dendrite RUN ./generate-keys --private-key matrix_key.pem diff --git a/clientapi/auth/password.go b/clientapi/auth/password.go index 700a72f5d..4de2b443c 100644 --- a/clientapi/auth/password.go +++ b/clientapi/auth/password.go @@ -61,7 +61,7 @@ func (t *LoginTypePassword) LoginFromJSON(ctx context.Context, reqBytes []byte) func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, *util.JSONResponse) { r := req.(*PasswordRequest) - username := strings.ToLower(r.Username()) + username := r.Username() if username == "" { return nil, &util.JSONResponse{ Code: http.StatusUnauthorized, @@ -74,32 +74,43 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, JSON: jsonerror.BadJSON("A password must be supplied."), } } - localpart, _, err := userutil.ParseUsernameParam(username, t.Config.Matrix) + localpart, domain, err := userutil.ParseUsernameParam(username, t.Config.Matrix) if err != nil { return nil, &util.JSONResponse{ Code: http.StatusUnauthorized, JSON: jsonerror.InvalidUsername(err.Error()), } } + if !t.Config.Matrix.IsLocalServerName(domain) { + return nil, &util.JSONResponse{ + Code: http.StatusUnauthorized, + JSON: jsonerror.InvalidUsername("The server name is not known."), + } + } // Squash username to all lowercase letters res := &api.QueryAccountByPasswordResponse{} - err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{Localpart: strings.ToLower(localpart), PlaintextPassword: r.Password}, res) + err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{ + Localpart: strings.ToLower(localpart), + ServerName: domain, + PlaintextPassword: r.Password, + }, res) if err != nil { return nil, &util.JSONResponse{ Code: http.StatusInternalServerError, - JSON: jsonerror.Unknown("unable to fetch account by password"), + JSON: jsonerror.Unknown("Unable to fetch account by password."), } } if !res.Exists { err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{ Localpart: localpart, + ServerName: domain, PlaintextPassword: r.Password, }, res) if err != nil { return nil, &util.JSONResponse{ Code: http.StatusInternalServerError, - JSON: jsonerror.Unknown("unable to fetch account by password"), + JSON: jsonerror.Unknown("Unable to fetch account by password."), } } // Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows diff --git a/clientapi/routing/admin.go b/clientapi/routing/admin.go index 69bca13be..be8073c33 100644 --- a/clientapi/routing/admin.go +++ b/clientapi/routing/admin.go @@ -102,6 +102,7 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userap if err != nil { return util.ErrorResponse(err) } + serverName := cfg.Matrix.ServerName localpart, ok := vars["localpart"] if !ok { return util.JSONResponse{ @@ -109,6 +110,9 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userap JSON: jsonerror.MissingArgument("Expecting user localpart."), } } + if l, s, err := cfg.Matrix.SplitLocalID('@', localpart); err == nil { + localpart, serverName = l, s + } request := struct { Password string `json:"password"` }{} @@ -126,6 +130,7 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userap } updateReq := &userapi.PerformPasswordUpdateRequest{ Localpart: localpart, + ServerName: serverName, Password: request.Password, LogoutDevices: true, } @@ -191,3 +196,43 @@ func AdminMarkAsStale(req *http.Request, cfg *config.ClientAPI, keyAPI api.Clien JSON: struct{}{}, } } + +func AdminDownloadState(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + roomID, ok := vars["roomID"] + if !ok { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.MissingArgument("Expecting room ID."), + } + } + serverName, ok := vars["serverName"] + if !ok { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.MissingArgument("Expecting remote server name."), + } + } + res := &roomserverAPI.PerformAdminDownloadStateResponse{} + if err := rsAPI.PerformAdminDownloadState( + req.Context(), + &roomserverAPI.PerformAdminDownloadStateRequest{ + UserID: device.UserID, + RoomID: roomID, + ServerName: gomatrixserverlib.ServerName(serverName), + }, + res, + ); err != nil { + return jsonerror.InternalAPIError(req.Context(), err) + } + if err := res.Error; err != nil { + return err.JSONResponse() + } + return util.JSONResponse{ + Code: 200, + JSON: map[string]interface{}{}, + } +} diff --git a/clientapi/routing/auth_fallback.go b/clientapi/routing/auth_fallback.go index abfe830fb..ad870993e 100644 --- a/clientapi/routing/auth_fallback.go +++ b/clientapi/routing/auth_fallback.go @@ -31,8 +31,7 @@ const recaptchaTemplate = ` Authentication - +