Add Helm CI stuff

This commit is contained in:
Till Faelligen 2022-12-05 08:23:00 +01:00
parent c355fe49fb
commit e5da7a527c
No known key found for this signature in database
GPG key ID: ACCDC9606D472758
13 changed files with 715 additions and 0 deletions

View file

@ -0,0 +1,45 @@
name: "Collect changes"
description: "Collects and stores changed files/charts"
outputs:
changesDetected:
description: "Whether or not changes to charts have been detected"
value: ${{ steps.filter.outputs.addedOrModified }}
addedOrModifiedFiles:
description: "A list of the files changed"
value: ${{ steps.filter.outputs.addedOrModified_files }}
addedOrModifiedCharts:
description: "A list of the charts changed"
value: ${{ steps.filter-charts.outputs.addedOrModified }}
runs:
using: "composite"
steps:
- name: Collect changed files
uses: dorny/paths-filter@v2
id: filter
with:
list-files: shell
filters: |
addedOrModified:
- added|modified: 'charts/*/**'
- name: Collect changed charts
if: |
steps.filter.outputs.addedOrModified == 'true'
id: filter-charts
shell: bash
run: |
CHARTS=()
PATHS=(${{ steps.filter.outputs.addedOrModified_files }})
# Get only the chart paths
for CHARTPATH in "${PATHS[@]}"
do
IFS='/' read -r -a path_parts <<< "${CHARTPATH}"
CHARTS+=("${path_parts[1]}/${path_parts[2]}")
done
# Remove duplicates
CHARTS=( `printf "%s\n" "${CHARTS[@]}" | sort -u` )
# Set output to changed charts
printf "::set-output name=addedOrModified::%s\n" "${CHARTS[*]}"

49
.github/scripts/check-releasenotes.sh vendored Executable file
View file

@ -0,0 +1,49 @@
#!/usr/bin/env bash
set -e
# Check if release notes have been changed
# Usage ./check-releasenotes.sh path
# require yq
command -v yq >/dev/null 2>&1 || {
printf >&2 "%s\n" "yq (https://github.com/mikefarah/yq) is not installed. Aborting."
exit 1
}
# Absolute path of repository
repository=$(git rev-parse --show-toplevel)
# Allow for a specific chart to be passed in as a argument
if [ $# -ge 1 ] && [ -n "$1" ]; then
root="$1"
chart_file="${1}/Chart.yaml"
if [ ! -f "$chart_file" ]; then
printf >&2 "File %s\n does not exist.\n" "${chart_file}"
exit 1
fi
cd $root
if [ -z "$DEFAULT_BRANCH" ]; then
DEFAULT_BRANCH=$(git remote show origin | awk '/HEAD branch/ {print $NF}')
fi
CURRENT=$(cat Chart.yaml | yq e '.annotations."artifacthub.io/changes"' -P -)
if [ "$CURRENT" == "" ] || [ "$CURRENT" == "null" ]; then
printf >&2 "Changelog annotation has not been set in %s!\n" "$chart_file"
exit 1
fi
DEFAULT_BRANCH=$(git remote show origin | awk '/HEAD branch/ {print $NF}')
ORIGINAL=$(git show origin/$DEFAULT_BRANCH:./Chart.yaml | yq e '.annotations."artifacthub.io/changes"' -P -)
if [ "$CURRENT" == "$ORIGINAL" ]; then
printf >&2 "Changelog annotation has not been updated in %s!\n" "$chart_file"
exit 1
fi
else
printf >&2 "%s\n" "No chart folder has been specified."
exit 1
fi

47
.github/scripts/gen-helm-docs.sh vendored Executable file
View file

@ -0,0 +1,47 @@
#!/usr/bin/env bash
set -eu
# Generate helm-docs for Helm charts
# Usage ./gen-helm-docs.sh [stable/incubator] [chart]
# require helm-docs
command -v helm-docs >/dev/null 2>&1 || {
echo >&2 "helm-docs (https://github.com/k8s-at-home/helm-docs) is not installed. Aborting."
exit 1
}
# Absolute path of repository
repository=$(git rev-parse --show-toplevel)
# Templates to copy into each chart directory
readme_template="${repository}/hack/templates/README.md.gotmpl"
readme_config_template="${repository}/hack/templates/README_CONFIG.md.gotmpl"
# Gather all charts using the common library, excluding common-test
charts=$(find "${repository}" -name "Chart.yaml")
# Allow for a specific chart to be passed in as a argument
if [ $# -ge 1 ] && [ -n "$1" ] && [ -n "$2" ]; then
charts="${repository}/charts/$1/$2/Chart.yaml"
root="$(dirname "${charts}")"
if [ ! -f "$charts" ]; then
echo "File ${charts} does not exist."
exit 1
fi
else
root="${repository}/charts/stable"
fi
for chart in ${charts}; do
chart_directory="$(dirname "${chart}")"
echo "-] Copying templates to ${chart_directory}"
# Copy CONFIG template to each Chart directory, do not overwrite if exists
cp -n "${readme_config_template}" "${chart_directory}" || true
done
# Run helm-docs for charts using the common library and the common library itself
helm-docs \
--ignore-file="${repository}/.helmdocsignore" \
--template-files="${readme_template}" \
--template-files="$(basename "${readme_config_template}")" \
--chart-search-root="${root}"

153
.github/scripts/renovate-releasenotes.py vendored Executable file
View file

@ -0,0 +1,153 @@
#!/usr/bin/env python
import os
import sys
import typer
from git import Repo
from loguru import logger
from pathlib import Path
from ruamel.yaml import YAML
from ruamel.yaml.comments import CommentedMap
from ruamel.yaml.scalarstring import LiteralScalarString
from typing import List
app = typer.Typer(add_completion=False)
def _setup_logging(debug):
"""
Setup the log formatter for this script
"""
log_level = "INFO"
if debug:
log_level = "DEBUG"
logger.remove()
logger.add(
sys.stdout,
colorize=True,
format="<level>{message}</level>",
level=log_level,
)
@app.command()
def main(
chart_folders: List[Path] = typer.Argument(
..., help="Folders containing the chart to process"),
check_branch: str = typer.Option(
None, help="The branch to compare against."),
chart_base_folder: Path = typer.Option(
"charts", help="The base folder where the charts reside."),
debug: bool = False,
):
_setup_logging(debug)
git_repository = Repo(search_parent_directories=True)
if check_branch:
logger.info(f"Trying to find branch {check_branch}...")
branch = next(
(ref for ref in git_repository.remotes.origin.refs if ref.name == check_branch),
None
)
else:
logger.info(f"Trying to determine default branch...")
branch = next(
(ref for ref in git_repository.remotes.origin.refs if ref.name == "origin/HEAD"),
None
)
if not branch:
logger.error(
f"Could not find branch {check_branch} to compare against.")
raise typer.Exit(1)
logger.info(f"Comparing against branch {branch}")
for chart_folder in chart_folders:
chart_folder = chart_base_folder.joinpath(chart_folder)
if not chart_folder.is_dir():
logger.error(f"Could not find folder {str(chart_folder)}")
raise typer.Exit(1)
chart_metadata_file = chart_folder.joinpath('Chart.yaml')
if not chart_metadata_file.is_file():
logger.error(f"Could not find file {str(chart_metadata_file)}")
raise typer.Exit(1)
logger.info(f"Updating changelog annotation for chart {chart_folder}")
yaml = YAML(typ=['rt', 'string'])
yaml.indent(mapping=2, sequence=4, offset=2)
yaml.explicit_start = True
yaml.preserve_quotes = True
yaml.width = 4096
old_chart_metadata = yaml.load(
git_repository.git.show(f"{branch}:{chart_metadata_file}")
)
new_chart_metadata = yaml.load(chart_metadata_file.read_text())
try:
old_chart_dependencies = old_chart_metadata["dependencies"]
except KeyError:
old_chart_dependencies = []
try:
new_chart_dependencies = new_chart_metadata["dependencies"]
except KeyError:
new_chart_dependencies = []
annotations = []
for dependency in new_chart_dependencies:
old_dep = None
if "alias" in dependency.keys():
old_dep = next(
(old_dep for old_dep in old_chart_dependencies if "alias" in old_dep.keys(
) and old_dep["alias"] == dependency["alias"]),
None
)
else:
old_dep = next(
(old_dep for old_dep in old_chart_dependencies if old_dep["name"] == dependency["name"]),
None
)
add_annotation = False
if old_dep:
if dependency["version"] != old_dep["version"]:
add_annotation = True
else:
add_annotation = True
if add_annotation:
if "alias" in dependency.keys():
annotations.append({
"kind": "changed",
"description": f"Upgraded `{dependency['name']}` chart dependency to version {dependency['version']} for alias '{dependency['alias']}'"
})
else:
annotations.append({
"kind": "changed",
"description": f"Upgraded `{dependency['name']}` chart dependency to version {dependency['version']}"
})
if annotations:
annotations = YAML(typ=['rt', 'string']
).dump_to_string(annotations)
if not "annotations" in new_chart_metadata:
new_chart_metadata["annotations"] = CommentedMap()
new_chart_metadata["annotations"]["artifacthub.io/changes"] = LiteralScalarString(
annotations)
yaml.dump(new_chart_metadata, chart_metadata_file)
if __name__ == "__main__":
app()

5
.github/scripts/requirements.txt vendored Normal file
View file

@ -0,0 +1,5 @@
GitPython==3.1.27
loguru==0.6.0
ruamel.yaml==0.17.21
ruamel.yaml.string==0.1.0
typer==0.6.1

View file

@ -2,12 +2,20 @@ name: Dendrite
on:
push:
paths-ignore:
- 'charts/**' # ignore helm chart changes
branches:
- main
pull_request:
paths-ignore:
- 'charts/**' # ignore helm chart changes
release:
paths-ignore:
- 'charts/**' # ignore helm chart changes
types: [published]
workflow_dispatch:
paths-ignore:
- 'charts/**' # ignore helm chart changes
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}

View file

@ -4,6 +4,8 @@ name: "Docker"
on:
release: # A GitHub release was published
paths-ignore:
- 'charts/**' # ignore helm chart changes
types: [published]
workflow_dispatch: # A build was manually requested
workflow_call: # Another pipeline called us

View file

@ -0,0 +1,81 @@
name: "Charts: Update README"
on:
workflow_call:
inputs:
modifiedCharts:
required: true
type: string
isRenovatePR:
required: true
type: string
outputs:
commitHash:
description: "The most recent commit hash at the end of this workflow"
value: ${{ jobs.generate-changelog.outputs.commitHash }}
jobs:
validate-changelog:
name: Validate changelog
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Check changelog annotations
if: inputs.isRenovatePR != 'true'
run: |
CHARTS=(${{ inputs.modifiedCharts }})
for i in "${CHARTS[@]}"
do
IFS='/' read -r -a chart_parts <<< "$i"
./.github/scripts/check-releasenotes.sh "charts/${chart_parts[0]}/${chart_parts[1]}"
echo ""
done
generate-changelog:
name: Generate changelog annotations
runs-on: ubuntu-latest
needs:
- validate-changelog
outputs:
commitHash: ${{ steps.save-commit-hash.outputs.commit_hash }}
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Annotate Charts.yaml for Renovate PR's
if: inputs.isRenovatePR == 'true'
env:
CHECK_BRANCH: "origin/${{ github.event.repository.default_branch }}"
run: |
pip install -r ./.github/scripts/requirements.txt
./.github/scripts/renovate-releasenotes.py --check-branch "$CHECK_BRANCH" ${{ inputs.modifiedCharts }}
- name: Create commit
id: create-commit
if: inputs.isRenovatePR == 'true'
uses: stefanzweifel/git-auto-commit-action@v4
with:
file_pattern: charts/**/
commit_message: "chore: Auto-update chart metadata"
commit_user_name: ${{ github.actor }}
commit_user_email: ${{ github.actor }}@users.noreply.github.com
- name: Save commit hash
id: save-commit-hash
run: |
if [ "${{ steps.create-commit.outputs.changes_detected || 'unknown' }}" == "true" ]; then
echo '::set-output name=commit_hash::${{ steps.create-commit.outputs.commit_hash }}'
else
echo "::set-output name=commit_hash::${GITHUB_SHA}"
fi

54
.github/workflows/helm-charts-lint.yaml vendored Normal file
View file

@ -0,0 +1,54 @@
name: "Charts: Lint"
on:
workflow_call:
inputs:
checkoutCommit:
required: true
type: string
chartChangesDetected:
required: true
type: string
jobs:
lint:
name: Lint charts
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ inputs.checkoutCommit }}
- name: Install Kubernetes tools
uses: yokawasa/action-setup-kube-tools@v0.8.2
with:
setup-tools: |
helmv3
helm: "3.8.0"
- uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Set up chart-testing
uses: helm/chart-testing-action@v2.3.0
- name: Collect changes
id: list-changed
if: inputs.chartChangesDetected == 'true'
run: |
EXCLUDED=$(yq eval -o=json '.excluded-charts // []' .github/ct-lint.yaml)
CHARTS=$(ct list-changed --config .github/ct-lint.yaml)
CHARTS_JSON=$(echo "${CHARTS}" | jq -R -s -c 'split("\n")[:-1]')
OUTPUT_JSON=$(echo "{\"excluded\": ${EXCLUDED}, \"all\": ${CHARTS_JSON}}" | jq -c '.all-.excluded')
echo ::set-output name=charts::${OUTPUT_JSON}
if [[ $(echo ${OUTPUT_JSON} | jq -c '. | length') -gt 0 ]]; then
echo "::set-output name=detected::true"
fi
- name: Run chart-testing (lint)
id: lint
if: steps.list-changed.outputs.detected == 'true'
run: ct lint --config .github/ct-lint.yaml

134
.github/workflows/helm-charts-test.yaml vendored Normal file
View file

@ -0,0 +1,134 @@
name: "Charts: Test"
on:
workflow_call:
inputs:
checkoutCommit:
required: true
type: string
chartChangesDetected:
required: true
type: string
jobs:
unit-test:
name: Run unit tests
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ inputs.checkoutCommit }}
- name: Install Kubernetes tools
uses: yokawasa/action-setup-kube-tools@v0.8.2
with:
setup-tools: |
helmv3
helm: "3.8.0"
- name: Install Ruby
uses: ruby/setup-ruby@v1
with:
ruby-version: 2.7
- name: Install dependencies
env:
RUBYJQ_USE_SYSTEM_LIBRARIES: 1
run: |
sudo apt-get update
sudo apt-get install libjq-dev
bundle install
- name: Run tests
run: |
bundle exec m -r ./test/
generate-install-matrix:
name: Generate matrix for install
runs-on: ubuntu-latest
outputs:
matrix: |
{
"chart": ${{ steps.list-changed.outputs.charts }}
}
detected: ${{ steps.list-changed.outputs.detected }}
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ inputs.checkoutCommit }}
- name: Set up chart-testing
uses: helm/chart-testing-action@v2.3.0
- name: Run chart-testing (list-changed)
id: list-changed
if: inputs.chartChangesDetected == 'true'
run: |
EXCLUDED=$(yq eval -o=json '.excluded-charts // []' .github/ct-install.yaml)
CHARTS=$(ct list-changed --config .github/ct-install.yaml)
CHARTS_JSON=$(echo "${CHARTS}" | jq -R -s -c 'split("\n")[:-1]')
OUTPUT_JSON=$(echo "{\"excluded\": ${EXCLUDED}, \"all\": ${CHARTS_JSON}}" | jq -c '.all-.excluded')
echo ::set-output name=charts::${OUTPUT_JSON}
if [[ $(echo ${OUTPUT_JSON} | jq -c '. | length') -gt 0 ]]; then
echo "::set-output name=detected::true"
fi
install-charts:
needs:
- generate-install-matrix
if: needs.generate-install-matrix.outputs.detected == 'true'
name: Install charts
strategy:
matrix: ${{ fromJson(needs.generate-install-matrix.outputs.matrix) }}
fail-fast: false
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ inputs.checkoutCommit }}
- name: Install Kubernetes tools
uses: yokawasa/action-setup-kube-tools@v0.8.2
with:
setup-tools: |
helmv3
helm: "3.6.3"
- uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Set up chart-testing
uses: helm/chart-testing-action@v2.3.0
- name: Create k3d cluster
uses: nolar/setup-k3d-k3s@v1
with:
version: v1.19
- name: Remove node taints
run: |
kubectl taint --all=true nodes node.cloudprovider.kubernetes.io/uninitialized- || true
- name: Run chart-testing (install)
run: ct install --config .github/ct-install.yaml --charts ${{ matrix.chart }}
# Summarize matrix https://github.community/t/status-check-for-a-matrix-jobs/127354/7
install_success:
needs:
- generate-install-matrix
- install-charts
if: |
always()
name: Install successful
runs-on: ubuntu-latest
steps:
- name: Check install matrix status
if: ${{ (needs.generate-install-matrix.outputs.detected == 'true') && (needs.install-charts.result != 'success') }}
run: exit 1

60
.github/workflows/helm-pr-metadata.yaml vendored Normal file
View file

@ -0,0 +1,60 @@
name: "Pull Request: Get metadata"
on:
workflow_call:
outputs:
isRenovatePR:
description: "Is the PR coming from Renovate?"
value: ${{ jobs.pr-metadata.outputs.isRenovatePR }}
isFork:
description: "Is the PR coming from a forked repo?"
value: ${{ jobs.pr-metadata.outputs.isFork }}
addedOrModified:
description: "Does the PR contain any changes?"
value: ${{ jobs.pr-changes.outputs.addedOrModified }}
addedOrModifiedFiles:
description: "A list of the files changed in this PR"
value: ${{ jobs.pr-changes.outputs.addedOrModifiedFiles }}
addedOrModifiedCharts:
description: "A list of the charts changed in this PR"
value: ${{ jobs.pr-changes.outputs.addedOrModifiedCharts }}
jobs:
pr-metadata:
name: Collect PR metadata
runs-on: ubuntu-latest
outputs:
isRenovatePR: ${{ startsWith(steps.branch-name.outputs.current_branch, 'renovate/') }}
isFork: ${{ github.event.pull_request.head.repo.full_name != github.repository }}
steps:
- name: Get branch name
id: branch-name
uses: tj-actions/branch-names@v5.4
- name: Save PR data to file
env:
PR_NUMBER: ${{ github.event.number }}
run: |
echo $PR_NUMBER > pr_number.txt
- name: Store pr data in artifact
uses: actions/upload-artifact@v3
with:
name: pr_metadata
path: ./pr_number.txt
retention-days: 5
pr-changes:
name: Collect PR changes
runs-on: ubuntu-latest
outputs:
addedOrModified: ${{ steps.collect-changes.outputs.changesDetected }}
addedOrModifiedFiles: ${{ steps.collect-changes.outputs.addedOrModifiedFiles }}
addedOrModifiedCharts: ${{ steps.collect-changes.outputs.addedOrModifiedCharts }}
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Collect changes
id: collect-changes
uses: ./.github/actions/collect-changes

View file

@ -0,0 +1,21 @@
name: "Pre-commit consistency check"
on:
workflow_call:
inputs:
modifiedFiles:
required: true
type: string
jobs:
pre-commit-check:
name: Run pre-commit checks
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Run against changes
uses: pre-commit/action@v3.0.0
with:
extra_args: --files ${{ inputs.modifiedFiles }}

56
.github/workflows/helm.yml vendored Normal file
View file

@ -0,0 +1,56 @@
name: "Pull Request: Validate"
on:
pull_request:
paths:
- 'charts/**' # only execute if we have helm chart changes
branches:
- main
types:
- opened
- edited
- reopened
- ready_for_review
- synchronize
concurrency:
group: ${{ github.head_ref }}-pr-validate
cancel-in-progress: true
jobs:
pr-metadata:
uses: matrix-org/dendrite/.github/workflows/helm-pr-metadata.yaml@main
pre-commit-check:
uses: matrix-org/dendrite/.github/workflows/helm-pre-commit-check.yaml@main
needs:
- pr-metadata
with:
modifiedFiles: ${{ needs.pr-metadata.outputs.addedOrModifiedFiles }}
charts-changelog:
uses: matrix-org/dendrite/.github/workflows/helm-charts-changelog.yaml@main
needs:
- pr-metadata
- pre-commit-check
with:
isRenovatePR: ${{ needs.pr-metadata.outputs.isRenovatePR }}
modifiedCharts: ${{ needs.pr-metadata.outputs.addedOrModifiedCharts }}
charts-lint:
uses: matrix-org/dendrite/.github/workflows/helm-charts-lint.yaml@main
needs:
- pr-metadata
- charts-changelog
with:
checkoutCommit: ${{ needs.charts-changelog.outputs.commitHash }}
chartChangesDetected: ${{ needs.pr-metadata.outputs.addedOrModified }}
charts-test:
uses: matrix-org/dendrite/.github/workflows/helm-charts-test.yaml@main
needs:
- pr-metadata
- charts-changelog
with:
checkoutCommit: ${{ needs.charts-changelog.outputs.commitHash }}
chartChangesDetected: ${{ needs.pr-metadata.outputs.addedOrModified }}