diff --git a/.github/actions/collect-changes/action.yaml b/.github/actions/collect-changes/action.yaml new file mode 100644 index 000000000..7f53c79d1 --- /dev/null +++ b/.github/actions/collect-changes/action.yaml @@ -0,0 +1,45 @@ +name: "Collect changes" +description: "Collects and stores changed files/charts" + +outputs: + changesDetected: + description: "Whether or not changes to charts have been detected" + value: ${{ steps.filter.outputs.addedOrModified }} + addedOrModifiedFiles: + description: "A list of the files changed" + value: ${{ steps.filter.outputs.addedOrModified_files }} + addedOrModifiedCharts: + description: "A list of the charts changed" + value: ${{ steps.filter-charts.outputs.addedOrModified }} + +runs: + using: "composite" + steps: + - name: Collect changed files + uses: dorny/paths-filter@v2 + id: filter + with: + list-files: shell + filters: | + addedOrModified: + - added|modified: 'charts/*/**' + + - name: Collect changed charts + if: | + steps.filter.outputs.addedOrModified == 'true' + id: filter-charts + shell: bash + run: | + CHARTS=() + PATHS=(${{ steps.filter.outputs.addedOrModified_files }}) + # Get only the chart paths + for CHARTPATH in "${PATHS[@]}" + do + IFS='/' read -r -a path_parts <<< "${CHARTPATH}" + CHARTS+=("${path_parts[1]}/${path_parts[2]}") + done + + # Remove duplicates + CHARTS=( `printf "%s\n" "${CHARTS[@]}" | sort -u` ) + # Set output to changed charts + printf "::set-output name=addedOrModified::%s\n" "${CHARTS[*]}" diff --git a/.github/scripts/check-releasenotes.sh b/.github/scripts/check-releasenotes.sh new file mode 100755 index 000000000..089a1699f --- /dev/null +++ b/.github/scripts/check-releasenotes.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -e + +# Check if release notes have been changed +# Usage ./check-releasenotes.sh path + +# require yq +command -v yq >/dev/null 2>&1 || { + printf >&2 "%s\n" "yq (https://github.com/mikefarah/yq) is not installed. Aborting." + exit 1 +} + +# Absolute path of repository +repository=$(git rev-parse --show-toplevel) + +# Allow for a specific chart to be passed in as a argument +if [ $# -ge 1 ] && [ -n "$1" ]; then + root="$1" + chart_file="${1}/Chart.yaml" + if [ ! -f "$chart_file" ]; then + printf >&2 "File %s\n does not exist.\n" "${chart_file}" + exit 1 + fi + + cd $root + + if [ -z "$DEFAULT_BRANCH" ]; then + DEFAULT_BRANCH=$(git remote show origin | awk '/HEAD branch/ {print $NF}') + fi + + CURRENT=$(cat Chart.yaml | yq e '.annotations."artifacthub.io/changes"' -P -) + + if [ "$CURRENT" == "" ] || [ "$CURRENT" == "null" ]; then + printf >&2 "Changelog annotation has not been set in %s!\n" "$chart_file" + exit 1 + fi + + DEFAULT_BRANCH=$(git remote show origin | awk '/HEAD branch/ {print $NF}') + ORIGINAL=$(git show origin/$DEFAULT_BRANCH:./Chart.yaml | yq e '.annotations."artifacthub.io/changes"' -P -) + + if [ "$CURRENT" == "$ORIGINAL" ]; then + printf >&2 "Changelog annotation has not been updated in %s!\n" "$chart_file" + exit 1 + fi +else + printf >&2 "%s\n" "No chart folder has been specified." + exit 1 +fi diff --git a/.github/scripts/gen-helm-docs.sh b/.github/scripts/gen-helm-docs.sh new file mode 100755 index 000000000..fca06603e --- /dev/null +++ b/.github/scripts/gen-helm-docs.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -eu + +# Generate helm-docs for Helm charts +# Usage ./gen-helm-docs.sh [stable/incubator] [chart] + +# require helm-docs +command -v helm-docs >/dev/null 2>&1 || { + echo >&2 "helm-docs (https://github.com/k8s-at-home/helm-docs) is not installed. Aborting." + exit 1 +} + +# Absolute path of repository +repository=$(git rev-parse --show-toplevel) + +# Templates to copy into each chart directory +readme_template="${repository}/hack/templates/README.md.gotmpl" +readme_config_template="${repository}/hack/templates/README_CONFIG.md.gotmpl" + +# Gather all charts using the common library, excluding common-test +charts=$(find "${repository}" -name "Chart.yaml") + +# Allow for a specific chart to be passed in as a argument +if [ $# -ge 1 ] && [ -n "$1" ] && [ -n "$2" ]; then + charts="${repository}/charts/$1/$2/Chart.yaml" + root="$(dirname "${charts}")" + if [ ! -f "$charts" ]; then + echo "File ${charts} does not exist." + exit 1 + fi +else + root="${repository}/charts/stable" +fi + +for chart in ${charts}; do + chart_directory="$(dirname "${chart}")" + echo "-] Copying templates to ${chart_directory}" + # Copy CONFIG template to each Chart directory, do not overwrite if exists + cp -n "${readme_config_template}" "${chart_directory}" || true +done + +# Run helm-docs for charts using the common library and the common library itself +helm-docs \ + --ignore-file="${repository}/.helmdocsignore" \ + --template-files="${readme_template}" \ + --template-files="$(basename "${readme_config_template}")" \ + --chart-search-root="${root}" diff --git a/.github/scripts/renovate-releasenotes.py b/.github/scripts/renovate-releasenotes.py new file mode 100755 index 000000000..058d9071f --- /dev/null +++ b/.github/scripts/renovate-releasenotes.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python + +import os +import sys +import typer + +from git import Repo +from loguru import logger +from pathlib import Path + +from ruamel.yaml import YAML +from ruamel.yaml.comments import CommentedMap +from ruamel.yaml.scalarstring import LiteralScalarString +from typing import List + +app = typer.Typer(add_completion=False) + + +def _setup_logging(debug): + """ + Setup the log formatter for this script + """ + + log_level = "INFO" + if debug: + log_level = "DEBUG" + + logger.remove() + logger.add( + sys.stdout, + colorize=True, + format="{message}", + level=log_level, + ) + + +@app.command() +def main( + chart_folders: List[Path] = typer.Argument( + ..., help="Folders containing the chart to process"), + check_branch: str = typer.Option( + None, help="The branch to compare against."), + chart_base_folder: Path = typer.Option( + "charts", help="The base folder where the charts reside."), + debug: bool = False, +): + _setup_logging(debug) + + git_repository = Repo(search_parent_directories=True) + + if check_branch: + logger.info(f"Trying to find branch {check_branch}...") + branch = next( + (ref for ref in git_repository.remotes.origin.refs if ref.name == check_branch), + None + ) + else: + logger.info(f"Trying to determine default branch...") + branch = next( + (ref for ref in git_repository.remotes.origin.refs if ref.name == "origin/HEAD"), + None + ) + + if not branch: + logger.error( + f"Could not find branch {check_branch} to compare against.") + raise typer.Exit(1) + + logger.info(f"Comparing against branch {branch}") + + for chart_folder in chart_folders: + chart_folder = chart_base_folder.joinpath(chart_folder) + if not chart_folder.is_dir(): + logger.error(f"Could not find folder {str(chart_folder)}") + raise typer.Exit(1) + + chart_metadata_file = chart_folder.joinpath('Chart.yaml') + + if not chart_metadata_file.is_file(): + logger.error(f"Could not find file {str(chart_metadata_file)}") + raise typer.Exit(1) + + logger.info(f"Updating changelog annotation for chart {chart_folder}") + + yaml = YAML(typ=['rt', 'string']) + yaml.indent(mapping=2, sequence=4, offset=2) + yaml.explicit_start = True + yaml.preserve_quotes = True + yaml.width = 4096 + + old_chart_metadata = yaml.load( + git_repository.git.show(f"{branch}:{chart_metadata_file}") + ) + new_chart_metadata = yaml.load(chart_metadata_file.read_text()) + + try: + old_chart_dependencies = old_chart_metadata["dependencies"] + except KeyError: + old_chart_dependencies = [] + + try: + new_chart_dependencies = new_chart_metadata["dependencies"] + except KeyError: + new_chart_dependencies = [] + + annotations = [] + for dependency in new_chart_dependencies: + old_dep = None + if "alias" in dependency.keys(): + old_dep = next( + (old_dep for old_dep in old_chart_dependencies if "alias" in old_dep.keys( + ) and old_dep["alias"] == dependency["alias"]), + None + ) + else: + old_dep = next( + (old_dep for old_dep in old_chart_dependencies if old_dep["name"] == dependency["name"]), + None + ) + + add_annotation = False + if old_dep: + if dependency["version"] != old_dep["version"]: + add_annotation = True + else: + add_annotation = True + + if add_annotation: + if "alias" in dependency.keys(): + annotations.append({ + "kind": "changed", + "description": f"Upgraded `{dependency['name']}` chart dependency to version {dependency['version']} for alias '{dependency['alias']}'" + }) + else: + annotations.append({ + "kind": "changed", + "description": f"Upgraded `{dependency['name']}` chart dependency to version {dependency['version']}" + }) + + if annotations: + annotations = YAML(typ=['rt', 'string'] + ).dump_to_string(annotations) + + if not "annotations" in new_chart_metadata: + new_chart_metadata["annotations"] = CommentedMap() + + new_chart_metadata["annotations"]["artifacthub.io/changes"] = LiteralScalarString( + annotations) + yaml.dump(new_chart_metadata, chart_metadata_file) + + +if __name__ == "__main__": + app() diff --git a/.github/scripts/requirements.txt b/.github/scripts/requirements.txt new file mode 100644 index 000000000..7bb21fe19 --- /dev/null +++ b/.github/scripts/requirements.txt @@ -0,0 +1,5 @@ +GitPython==3.1.27 +loguru==0.6.0 +ruamel.yaml==0.17.21 +ruamel.yaml.string==0.1.0 +typer==0.6.1 diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index 593012ef3..6bbd41b0f 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -2,12 +2,20 @@ name: Dendrite on: push: + paths-ignore: + - 'charts/**' # ignore helm chart changes branches: - main pull_request: + paths-ignore: + - 'charts/**' # ignore helm chart changes release: + paths-ignore: + - 'charts/**' # ignore helm chart changes types: [published] workflow_dispatch: + paths-ignore: + - 'charts/**' # ignore helm chart changes concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2e17539d8..0365e1e6d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,6 +4,8 @@ name: "Docker" on: release: # A GitHub release was published + paths-ignore: + - 'charts/**' # ignore helm chart changes types: [published] workflow_dispatch: # A build was manually requested workflow_call: # Another pipeline called us diff --git a/.github/workflows/helm-charts-changelog.yaml b/.github/workflows/helm-charts-changelog.yaml new file mode 100644 index 000000000..e02931773 --- /dev/null +++ b/.github/workflows/helm-charts-changelog.yaml @@ -0,0 +1,81 @@ +name: "Charts: Update README" + +on: + workflow_call: + inputs: + modifiedCharts: + required: true + type: string + isRenovatePR: + required: true + type: string + outputs: + commitHash: + description: "The most recent commit hash at the end of this workflow" + value: ${{ jobs.generate-changelog.outputs.commitHash }} + +jobs: + validate-changelog: + name: Validate changelog + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Check changelog annotations + if: inputs.isRenovatePR != 'true' + run: | + CHARTS=(${{ inputs.modifiedCharts }}) + for i in "${CHARTS[@]}" + do + IFS='/' read -r -a chart_parts <<< "$i" + ./.github/scripts/check-releasenotes.sh "charts/${chart_parts[0]}/${chart_parts[1]}" + echo "" + done + + generate-changelog: + name: Generate changelog annotations + runs-on: ubuntu-latest + needs: + - validate-changelog + outputs: + commitHash: ${{ steps.save-commit-hash.outputs.commit_hash }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Annotate Charts.yaml for Renovate PR's + if: inputs.isRenovatePR == 'true' + env: + CHECK_BRANCH: "origin/${{ github.event.repository.default_branch }}" + run: | + pip install -r ./.github/scripts/requirements.txt + ./.github/scripts/renovate-releasenotes.py --check-branch "$CHECK_BRANCH" ${{ inputs.modifiedCharts }} + + - name: Create commit + id: create-commit + if: inputs.isRenovatePR == 'true' + uses: stefanzweifel/git-auto-commit-action@v4 + with: + file_pattern: charts/**/ + commit_message: "chore: Auto-update chart metadata" + commit_user_name: ${{ github.actor }} + commit_user_email: ${{ github.actor }}@users.noreply.github.com + + - name: Save commit hash + id: save-commit-hash + run: | + if [ "${{ steps.create-commit.outputs.changes_detected || 'unknown' }}" == "true" ]; then + echo '::set-output name=commit_hash::${{ steps.create-commit.outputs.commit_hash }}' + else + echo "::set-output name=commit_hash::${GITHUB_SHA}" + fi diff --git a/.github/workflows/helm-charts-lint.yaml b/.github/workflows/helm-charts-lint.yaml new file mode 100644 index 000000000..82be78f08 --- /dev/null +++ b/.github/workflows/helm-charts-lint.yaml @@ -0,0 +1,54 @@ +name: "Charts: Lint" + +on: + workflow_call: + inputs: + checkoutCommit: + required: true + type: string + chartChangesDetected: + required: true + type: string + +jobs: + lint: + name: Lint charts + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.checkoutCommit }} + + - name: Install Kubernetes tools + uses: yokawasa/action-setup-kube-tools@v0.8.2 + with: + setup-tools: | + helmv3 + helm: "3.8.0" + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.3.0 + + - name: Collect changes + id: list-changed + if: inputs.chartChangesDetected == 'true' + run: | + EXCLUDED=$(yq eval -o=json '.excluded-charts // []' .github/ct-lint.yaml) + CHARTS=$(ct list-changed --config .github/ct-lint.yaml) + CHARTS_JSON=$(echo "${CHARTS}" | jq -R -s -c 'split("\n")[:-1]') + OUTPUT_JSON=$(echo "{\"excluded\": ${EXCLUDED}, \"all\": ${CHARTS_JSON}}" | jq -c '.all-.excluded') + echo ::set-output name=charts::${OUTPUT_JSON} + if [[ $(echo ${OUTPUT_JSON} | jq -c '. | length') -gt 0 ]]; then + echo "::set-output name=detected::true" + fi + + - name: Run chart-testing (lint) + id: lint + if: steps.list-changed.outputs.detected == 'true' + run: ct lint --config .github/ct-lint.yaml diff --git a/.github/workflows/helm-charts-test.yaml b/.github/workflows/helm-charts-test.yaml new file mode 100644 index 000000000..a8511f727 --- /dev/null +++ b/.github/workflows/helm-charts-test.yaml @@ -0,0 +1,134 @@ +name: "Charts: Test" + +on: + workflow_call: + inputs: + checkoutCommit: + required: true + type: string + chartChangesDetected: + required: true + type: string + +jobs: + unit-test: + name: Run unit tests + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.checkoutCommit }} + + - name: Install Kubernetes tools + uses: yokawasa/action-setup-kube-tools@v0.8.2 + with: + setup-tools: | + helmv3 + helm: "3.8.0" + + - name: Install Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: 2.7 + + - name: Install dependencies + env: + RUBYJQ_USE_SYSTEM_LIBRARIES: 1 + run: | + sudo apt-get update + sudo apt-get install libjq-dev + bundle install + + - name: Run tests + run: | + bundle exec m -r ./test/ + + generate-install-matrix: + name: Generate matrix for install + runs-on: ubuntu-latest + outputs: + matrix: | + { + "chart": ${{ steps.list-changed.outputs.charts }} + } + detected: ${{ steps.list-changed.outputs.detected }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.checkoutCommit }} + + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.3.0 + + - name: Run chart-testing (list-changed) + id: list-changed + if: inputs.chartChangesDetected == 'true' + run: | + EXCLUDED=$(yq eval -o=json '.excluded-charts // []' .github/ct-install.yaml) + CHARTS=$(ct list-changed --config .github/ct-install.yaml) + CHARTS_JSON=$(echo "${CHARTS}" | jq -R -s -c 'split("\n")[:-1]') + OUTPUT_JSON=$(echo "{\"excluded\": ${EXCLUDED}, \"all\": ${CHARTS_JSON}}" | jq -c '.all-.excluded') + echo ::set-output name=charts::${OUTPUT_JSON} + if [[ $(echo ${OUTPUT_JSON} | jq -c '. | length') -gt 0 ]]; then + echo "::set-output name=detected::true" + fi + + install-charts: + needs: + - generate-install-matrix + if: needs.generate-install-matrix.outputs.detected == 'true' + name: Install charts + strategy: + matrix: ${{ fromJson(needs.generate-install-matrix.outputs.matrix) }} + fail-fast: false + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.checkoutCommit }} + + - name: Install Kubernetes tools + uses: yokawasa/action-setup-kube-tools@v0.8.2 + with: + setup-tools: | + helmv3 + helm: "3.6.3" + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.3.0 + + - name: Create k3d cluster + uses: nolar/setup-k3d-k3s@v1 + with: + version: v1.19 + + - name: Remove node taints + run: | + kubectl taint --all=true nodes node.cloudprovider.kubernetes.io/uninitialized- || true + + - name: Run chart-testing (install) + run: ct install --config .github/ct-install.yaml --charts ${{ matrix.chart }} + + # Summarize matrix https://github.community/t/status-check-for-a-matrix-jobs/127354/7 + install_success: + needs: + - generate-install-matrix + - install-charts + if: | + always() + name: Install successful + runs-on: ubuntu-latest + steps: + - name: Check install matrix status + if: ${{ (needs.generate-install-matrix.outputs.detected == 'true') && (needs.install-charts.result != 'success') }} + run: exit 1 diff --git a/.github/workflows/helm-pr-metadata.yaml b/.github/workflows/helm-pr-metadata.yaml new file mode 100644 index 000000000..0496a1854 --- /dev/null +++ b/.github/workflows/helm-pr-metadata.yaml @@ -0,0 +1,60 @@ +name: "Pull Request: Get metadata" + +on: + workflow_call: + outputs: + isRenovatePR: + description: "Is the PR coming from Renovate?" + value: ${{ jobs.pr-metadata.outputs.isRenovatePR }} + isFork: + description: "Is the PR coming from a forked repo?" + value: ${{ jobs.pr-metadata.outputs.isFork }} + addedOrModified: + description: "Does the PR contain any changes?" + value: ${{ jobs.pr-changes.outputs.addedOrModified }} + addedOrModifiedFiles: + description: "A list of the files changed in this PR" + value: ${{ jobs.pr-changes.outputs.addedOrModifiedFiles }} + addedOrModifiedCharts: + description: "A list of the charts changed in this PR" + value: ${{ jobs.pr-changes.outputs.addedOrModifiedCharts }} + +jobs: + pr-metadata: + name: Collect PR metadata + runs-on: ubuntu-latest + outputs: + isRenovatePR: ${{ startsWith(steps.branch-name.outputs.current_branch, 'renovate/') }} + isFork: ${{ github.event.pull_request.head.repo.full_name != github.repository }} + steps: + - name: Get branch name + id: branch-name + uses: tj-actions/branch-names@v5.4 + + - name: Save PR data to file + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + + - name: Store pr data in artifact + uses: actions/upload-artifact@v3 + with: + name: pr_metadata + path: ./pr_number.txt + retention-days: 5 + + pr-changes: + name: Collect PR changes + runs-on: ubuntu-latest + outputs: + addedOrModified: ${{ steps.collect-changes.outputs.changesDetected }} + addedOrModifiedFiles: ${{ steps.collect-changes.outputs.addedOrModifiedFiles }} + addedOrModifiedCharts: ${{ steps.collect-changes.outputs.addedOrModifiedCharts }} + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Collect changes + id: collect-changes + uses: ./.github/actions/collect-changes diff --git a/.github/workflows/helm-pre-commit-check.yaml b/.github/workflows/helm-pre-commit-check.yaml new file mode 100644 index 000000000..f5b558635 --- /dev/null +++ b/.github/workflows/helm-pre-commit-check.yaml @@ -0,0 +1,21 @@ +name: "Pre-commit consistency check" + +on: + workflow_call: + inputs: + modifiedFiles: + required: true + type: string + +jobs: + pre-commit-check: + name: Run pre-commit checks + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Run against changes + uses: pre-commit/action@v3.0.0 + with: + extra_args: --files ${{ inputs.modifiedFiles }} diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml new file mode 100644 index 000000000..a1a1bfe41 --- /dev/null +++ b/.github/workflows/helm.yml @@ -0,0 +1,56 @@ +name: "Pull Request: Validate" + +on: + pull_request: + paths: + - 'charts/**' # only execute if we have helm chart changes + branches: + - main + types: + - opened + - edited + - reopened + - ready_for_review + - synchronize + +concurrency: + group: ${{ github.head_ref }}-pr-validate + cancel-in-progress: true + +jobs: + pr-metadata: + uses: matrix-org/dendrite/.github/workflows/helm-pr-metadata.yaml@main + + pre-commit-check: + uses: matrix-org/dendrite/.github/workflows/helm-pre-commit-check.yaml@main + needs: + - pr-metadata + with: + modifiedFiles: ${{ needs.pr-metadata.outputs.addedOrModifiedFiles }} + + charts-changelog: + uses: matrix-org/dendrite/.github/workflows/helm-charts-changelog.yaml@main + needs: + - pr-metadata + - pre-commit-check + with: + isRenovatePR: ${{ needs.pr-metadata.outputs.isRenovatePR }} + modifiedCharts: ${{ needs.pr-metadata.outputs.addedOrModifiedCharts }} + + charts-lint: + uses: matrix-org/dendrite/.github/workflows/helm-charts-lint.yaml@main + needs: + - pr-metadata + - charts-changelog + with: + checkoutCommit: ${{ needs.charts-changelog.outputs.commitHash }} + chartChangesDetected: ${{ needs.pr-metadata.outputs.addedOrModified }} + + charts-test: + uses: matrix-org/dendrite/.github/workflows/helm-charts-test.yaml@main + needs: + - pr-metadata + - charts-changelog + with: + checkoutCommit: ${{ needs.charts-changelog.outputs.commitHash }} + chartChangesDetected: ${{ needs.pr-metadata.outputs.addedOrModified }} \ No newline at end of file