From cc1b348ef3512516cf7364be7d4066234353c804 Mon Sep 17 00:00:00 2001 From: Samuel <15628653+swibrow@users.noreply.github.com> Date: Fri, 30 Jan 2026 08:19:17 +0100 Subject: [PATCH 1/2] feat: add service pipeline --- .github/workflows/_test-service-pipeline.yaml | 120 ++++++++ .github/workflows/service-pipeline.yaml | 263 ++++++++++++++++++ docs/workflows/lambda-python.md | 8 + docs/workflows/service-pipeline.md | 192 +++++++++++++ tests/service-pipeline/Dockerfile | 7 + tests/service-pipeline/matrix-config.yaml | 8 + tests/service-pipeline/secrets.Dockerfile | 12 + 7 files changed, 610 insertions(+) create mode 100644 .github/workflows/_test-service-pipeline.yaml create mode 100644 .github/workflows/service-pipeline.yaml create mode 100644 docs/workflows/service-pipeline.md create mode 100644 tests/service-pipeline/Dockerfile create mode 100644 tests/service-pipeline/matrix-config.yaml create mode 100644 tests/service-pipeline/secrets.Dockerfile diff --git a/.github/workflows/_test-service-pipeline.yaml b/.github/workflows/_test-service-pipeline.yaml new file mode 100644 index 00000000..026bb898 --- /dev/null +++ b/.github/workflows/_test-service-pipeline.yaml @@ -0,0 +1,120 @@ +on: + pull_request: + branches: + - main + paths: + - ".github/workflows/_test-service-pipeline.yaml" + - ".github/workflows/service-pipeline.yaml" + - "tests/service-pipeline/**" + +jobs: + test_build: + name: Test Build Stage + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Prepare image metadata + id: meta + run: | + SERVICE="test-service-pipeline" + SHORT_SHA="${GITHUB_SHA:0:7}" + + TAGS="${SHORT_SHA}" + DOCKER_TAGS="${SERVICE}:${SHORT_SHA}" + + if [ "${{ github.event_name }}" = "pull_request" ]; then + PR_TAG="pr-${{ github.event.pull_request.number }}" + TAGS="${TAGS} + ${PR_TAG}" + DOCKER_TAGS="${DOCKER_TAGS},${SERVICE}:${PR_TAG}" + fi + + echo "sha=${SHORT_SHA}" >> $GITHUB_OUTPUT + echo "tags<> $GITHUB_OUTPUT + echo "${TAGS}" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + echo "docker_tags=${DOCKER_TAGS}" >> $GITHUB_OUTPUT + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build image + uses: docker/build-push-action@v6 + with: + context: ./tests/service-pipeline + file: ./tests/service-pipeline/Dockerfile + tags: ${{ steps.meta.outputs.docker_tags }} + outputs: type=docker,dest=/tmp/test-service-pipeline.tar + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Upload artifact + uses: actions/upload-artifact@v5 + with: + name: test-service-pipeline-image + path: /tmp/test-service-pipeline.tar + retention-days: 1 + + - name: Verify artifact + run: | + docker load --input /tmp/test-service-pipeline.tar + docker run --rm test-service-pipeline:${{ steps.meta.outputs.sha }} | grep "service-pipeline test" + echo "✅ Build stage works correctly" + + test_setup: + name: Test Setup Stage + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Load configuration + id: config + uses: DND-IT/action-config@v1 + with: + config-path: tests/service-pipeline/matrix-config.yaml + + - name: Validate matrix output + run: | + MATRIX='${{ steps.config.outputs.matrix }}' + echo "Matrix output: ${MATRIX}" + + if [ -z "$MATRIX" ] || [ "$MATRIX" = "null" ]; then + echo "❌ Matrix output is empty" + exit 1 + fi + + echo "✅ Setup stage works correctly" + + test_build_with_secrets: + name: Test Build with Secrets + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build image with secrets + uses: docker/build-push-action@v6 + with: + context: ./tests/service-pipeline + file: ./tests/service-pipeline/secrets.Dockerfile + tags: test-service-pipeline-secrets:latest + outputs: type=docker,dest=/tmp/test-secrets.tar + secrets: | + SECRET_TOKEN=my-secret-value + build-args: | + CUSTOM_ARG=hello-from-build-arg + + - name: Verify secret was mounted + run: | + docker load --input /tmp/test-secrets.tar + OUTPUT=$(docker run --rm test-service-pipeline-secrets:latest) + echo "Output: ${OUTPUT}" + echo "${OUTPUT}" | grep "my-secret-value" || { echo "❌ Secret not mounted"; exit 1; } + echo "${OUTPUT}" | grep "hello-from-build-arg" || { echo "❌ Build arg not passed"; exit 1; } + echo "✅ Secrets and build args work correctly" diff --git a/.github/workflows/service-pipeline.yaml b/.github/workflows/service-pipeline.yaml new file mode 100644 index 00000000..cc5d463b --- /dev/null +++ b/.github/workflows/service-pipeline.yaml @@ -0,0 +1,263 @@ +name: Service Pipeline + +on: + workflow_call: + inputs: + service_name: + description: "Name of the service / ECR repository (e.g., social-media)" + required: true + type: string + service_path: + description: "Path to service directory containing the Dockerfile (e.g., backend)" + required: false + type: string + default: '.' + dockerfile_path: + description: "Path to the Dockerfile relative to service_path. Defaults to Dockerfile." + required: false + type: string + config_path: + description: "Path to the matrix config file with AWS environments" + required: true + type: string + default: '.github/matrix-config.yaml' + validate_pr_title: + description: "Validate PR title follows conventional commit format" + required: false + type: boolean + default: true + docker_build_args: + description: "Additional Docker build arguments (newline-separated KEY=VALUE pairs)" + required: false + type: string + default: '' + slack_channel: + description: "Slack channel ID for release notifications (empty to skip)" + required: false + type: string + default: '' + app_id: + description: "GitHub App ID for semantic-release authentication" + required: true + type: string + outputs: + new_release_published: + description: "Whether a new release was published ('true' or 'false')" + value: ${{ jobs.release-info.outputs.new_release_published }} + new_release_version: + description: "Version of the new release (e.g. 1.3.0)" + value: ${{ jobs.release-info.outputs.new_release_version }} + secrets: + docker_secrets: + description: "Docker build secrets (newline-separated id=value pairs)" + required: false + slack_bot_token: + required: false + app_private_key: + required: true + +concurrency: + group: service-pipeline-${{ inputs.service_name }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + validate-pr-title: + name: Validate PR Title + if: github.event_name == 'pull_request' && inputs.validate_pr_title + runs-on: ubuntu-latest + steps: + - name: Validate conventional commit format + uses: amannn/action-semantic-pull-request@v6 + env: + GITHUB_TOKEN: ${{ github.token }} + + setup: + name: Setup + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.config.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Load configuration + id: config + uses: DND-IT/action-config@v1 + with: + config-path: ${{ inputs.config_path }} + + build: + name: Build + runs-on: ubuntu-latest + outputs: + sha: ${{ steps.meta.outputs.sha }} + image-tags: ${{ steps.meta.outputs.tags }} + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Prepare image metadata + id: meta + run: | + SERVICE="${{ inputs.service_name }}" + FULL_SHA="${{ github.sha }}" + SHORT_SHA="${FULL_SHA:0:7}" + + TAGS="${SHORT_SHA}" + + if [ "${{ github.event_name }}" = "pull_request" ]; then + TAGS="${TAGS} + pr-${{ github.event.pull_request.number }}" + fi + + DOCKER_TAGS=$(echo "${TAGS}" | while read -r tag; do + [ -n "$tag" ] && echo "${SERVICE}:${tag}" + done | paste -sd "," -) + + echo "sha=${SHORT_SHA}" >> $GITHUB_OUTPUT + echo "tags<> $GITHUB_OUTPUT + echo "${TAGS}" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + echo "docker_tags=${DOCKER_TAGS}" >> $GITHUB_OUTPUT + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and export + uses: docker/build-push-action@v6 + env: + DOCKERFILE: ${{ inputs.dockerfile_path && format('{0}/{1}', inputs.service_path, inputs.dockerfile_path) || format('{0}/Dockerfile', inputs.service_path) }} + with: + context: ./${{ inputs.service_path }} + file: ${{ env.DOCKERFILE }} + tags: ${{ steps.meta.outputs.docker_tags }} + outputs: type=docker,dest=/tmp/${{ inputs.service_name }}.tar + secrets: ${{ secrets.docker_secrets }} + build-args: | + VERSION=${{ steps.meta.outputs.sha }} + BUILD_DATE=${{ github.event.head_commit.timestamp }} + VCS_REF=${{ github.sha }} + ${{ inputs.docker_build_args }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Upload artifact + uses: actions/upload-artifact@v5 + with: + name: ${{ inputs.service_name }}-image + path: /tmp/${{ inputs.service_name }}.tar + retention-days: 1 + + release-info: + name: Release Info + if: github.ref_name == github.event.repository.default_branch && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + permissions: + contents: write + issues: write + pull-requests: write + uses: DND-IT/github-workflows/.github/workflows/gh-release.yaml@v3 + with: + use_semantic_release: true + dry_run: true + app_id: ${{ inputs.app_id }} + working_directory: ${{ inputs.service_path }} + secrets: + app_private_key: ${{ secrets.app_private_key }} + + push: + name: Push (${{ matrix.environment }}) + needs: [setup, build, release-info] + if: always() && needs.build.result == 'success' + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.setup.outputs.matrix) }} + steps: + - name: Download artifact + uses: actions/download-artifact@v6 + with: + name: ${{ inputs.service_name }}-image + path: /tmp + + - name: Load image + run: docker load --input /tmp/${{ inputs.service_name }}.tar + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v5 + with: + role-to-assume: arn:aws:iam::${{ matrix.aws_account_id }}:role/${{ matrix.aws_iam_role_name }} + aws-region: ${{ matrix.aws_region }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Tag and push to ECR + run: | + ECR_REGISTRY="${{ steps.login-ecr.outputs.registry }}" + SERVICE="${{ inputs.service_name }}" + SHA="${{ needs.build.outputs.sha }}" + ALL_TAGS="${{ needs.build.outputs.image-tags }}" + + # Tag all build tags + while IFS= read -r tag; do + [ -n "$tag" ] && docker tag "${SERVICE}:${tag}" "${ECR_REGISTRY}/${SERVICE}:${tag}" + done <<< "${ALL_TAGS}" + + # Tag 'latest' on default branch + if [ "${{ github.event_name }}" = "push" ] && [ "${{ github.ref }}" = "refs/heads/${{ github.event.repository.default_branch }}" ]; then + docker tag "${SERVICE}:${SHA}" "${ECR_REGISTRY}/${SERVICE}:latest" + fi + + # Tag version if semantic release found a new version + if [ "${{ needs.release-info.outputs.new_release_published }}" = "true" ]; then + VERSION="${{ needs.release-info.outputs.new_release_version }}" + docker tag "${SERVICE}:${SHA}" "${ECR_REGISTRY}/${SERVICE}:${VERSION}" + fi + + docker push --all-tags "${ECR_REGISTRY}/${SERVICE}" + + release: + name: Create Release + needs: [push, release-info] + if: needs.release-info.outputs.new_release_published == 'true' + permissions: + contents: write + issues: write + pull-requests: write + uses: DND-IT/github-workflows/.github/workflows/gh-release.yaml@v3 + with: + use_semantic_release: true + app_id: ${{ inputs.app_id }} + working_directory: ${{ inputs.service_path }} + secrets: + app_private_key: ${{ secrets.app_private_key }} + + notify: + name: Notify Slack + needs: [release, release-info] + if: | + always() && + needs.release.result == 'success' && + needs.release-info.outputs.new_release_published == 'true' && + inputs.slack_channel != '' + uses: DND-IT/github-workflows/.github/workflows/notify-slack.yaml@v3 + secrets: + slack_bot_token: ${{ secrets.slack_bot_token }} + with: + channel: ${{ inputs.slack_channel }} + notification_title: "New ${{ inputs.service_name }} release" + notification_message: "${{ inputs.service_name }} v${{ needs.release-info.outputs.new_release_version }} has been released!" + status: "success" + additional_fields: | + [ + {"name": "Service", "value": "${{ inputs.service_name }}"}, + {"name": "Version", "value": "v${{ needs.release-info.outputs.new_release_version }}"}, + {"name": "Release", "value": "<${{ github.server_url }}/${{ github.repository }}/releases/tag/v${{ needs.release-info.outputs.new_release_version }}|v${{ needs.release-info.outputs.new_release_version }}>"} + ] + include_workflow_link: true + include_triggered_by: true diff --git a/docs/workflows/lambda-python.md b/docs/workflows/lambda-python.md index 514f25a5..0006cb5f 100644 --- a/docs/workflows/lambda-python.md +++ b/docs/workflows/lambda-python.md @@ -20,6 +20,7 @@ Usefull to deploy an AWS lambda function or layer. | --- | --- | --- | --- | --- | | `python_version` |

Python version. Check https://github.com/actions/setup-python for valid values

| `string` | `false` | `3.12` | | `source_dir` |

Directory of the Python source code. Should contain the requirements.txt file

| `string` | `true` | `""` | +| `requirements_filepath` |

Path to the requirements.txt file, relative to source_dir

| `string` | `false` | `requirements.txt` | | `zip_filename` |

The zip file to create. It's relativ to the repository root

| `string` | `false` | `python_package.zip` | | `gh_artifact_name` |

Name of the artifact to upload

| `string` | `true` | `""` | | `gh_artifact_retention_days` |

Number of days to retain the artifact

| `number` | `false` | `30` | @@ -51,6 +52,13 @@ jobs: # Required: true # Default: "" + requirements_filepath: + # Path to the requirements.txt file, relative to source_dir + # + # Type: string + # Required: false + # Default: requirements.txt + zip_filename: # The zip file to create. It's relativ to the repository root # diff --git a/docs/workflows/service-pipeline.md b/docs/workflows/service-pipeline.md new file mode 100644 index 00000000..259cbc77 --- /dev/null +++ b/docs/workflows/service-pipeline.md @@ -0,0 +1,192 @@ +--- +title: Service Pipeline +--- + +## Description + +End-to-end build and release pipeline for containerized services. Builds a Docker image, pushes to ECR across multiple AWS environments, creates a GitHub release via semantic-release, and sends a Slack notification. + +**Pipeline flow:** Setup → Build → Release Info (dry-run) → Push (per environment) → Create Release → Notify Slack + + + + + + + +## Prerequisites + +- A `.releaserc.json` at the repo root (or `service_path`) excluding the `@semantic-release/npm` plugin. See [gh-release](./gh-release.md) for details. +- A `.github/matrix-config.yaml` defining AWS environments. See [action-config](https://github.com/DND-IT/action-config) for the format. +- Repository secrets: `FISSION_GH_APP_PRIVATE_KEY`, `SLACK_BOT_TOKEN` +- Repository variables: `FISSION_GH_APP_ID` + +## Examples + +### Minimal — single service repo + +```yaml +name: backend + +on: + push: + branches: [main] + paths: ['backend/**'] + pull_request: + paths: ['backend/**'] + workflow_dispatch: + +jobs: + pipeline: + uses: DND-IT/github-workflows/.github/workflows/service-pipeline.yaml@v3 + with: + service_name: social-media + service_path: backend + config_path: '.github/matrix-config.yaml' + app_id: ${{ vars.FISSION_GH_APP_ID }} + secrets: + app_private_key: ${{ secrets.FISSION_GH_APP_PRIVATE_KEY }} +``` + +### With Slack notifications + +```yaml +jobs: + pipeline: + uses: DND-IT/github-workflows/.github/workflows/service-pipeline.yaml@v3 + with: + service_name: my-api + service_path: . + config_path: '.github/matrix-config.yaml' + slack_channel: 'C09LG2L8EQ5' + app_id: ${{ vars.FISSION_GH_APP_ID }} + secrets: + slack_bot_token: ${{ secrets.SLACK_BOT_TOKEN }} + app_private_key: ${{ secrets.FISSION_GH_APP_PRIVATE_KEY }} +``` + +### Custom Dockerfile path + +```yaml +jobs: + pipeline: + uses: DND-IT/github-workflows/.github/workflows/service-pipeline.yaml@v3 + with: + service_name: my-service + service_path: services/api + dockerfile_path: docker/production.Dockerfile + config_path: '.github/matrix-config.yaml' + app_id: ${{ vars.FISSION_GH_APP_ID }} + secrets: + app_private_key: ${{ secrets.FISSION_GH_APP_PRIVATE_KEY }} +``` + +### With Docker build secrets + +Use `docker_secrets` to pass secrets into the Docker build (e.g., private registry tokens). In your Dockerfile, mount them with `RUN --mount=type=secret`: + +```yaml +jobs: + pipeline: + uses: DND-IT/github-workflows/.github/workflows/service-pipeline.yaml@v3 + with: + service_name: my-service + config_path: '.github/matrix-config.yaml' + app_id: ${{ vars.FISSION_GH_APP_ID }} + secrets: + app_private_key: ${{ secrets.FISSION_GH_APP_PRIVATE_KEY }} + docker_secrets: | + NPM_TOKEN=${{ secrets.NPM_TOKEN }} + PIP_INDEX_URL=${{ secrets.PIP_INDEX_URL }} +``` + +Then in your `Dockerfile`: + +```dockerfile +RUN --mount=type=secret,id=NPM_TOKEN \ + NPM_TOKEN=$(cat /run/secrets/NPM_TOKEN) npm ci +``` + +### With custom build arguments + +Use `docker_build_args` to pass additional build arguments beyond the built-in `VERSION`, `BUILD_DATE`, and `VCS_REF`: + +```yaml +jobs: + pipeline: + uses: DND-IT/github-workflows/.github/workflows/service-pipeline.yaml@v3 + with: + service_name: my-service + config_path: '.github/matrix-config.yaml' + app_id: ${{ vars.FISSION_GH_APP_ID }} + docker_build_args: | + NODE_ENV=production + API_BASE_URL=https://api.example.com + secrets: + app_private_key: ${{ secrets.FISSION_GH_APP_PRIVATE_KEY }} +``` + +### Matrix config example + +The `config_path` file should follow the [action-config](https://github.com/DND-IT/action-config) format: + +```yaml +environments: + - dev + - prod + +config: + dev: + aws_account_id: "123456789012" + aws_region: eu-central-1 + aws_iam_role_name: cicd-iac + prod: + aws_account_id: "987654321098" + aws_region: eu-central-1 + aws_iam_role_name: cicd-iac +``` + +### `.releaserc.json` example + +```json +{ + "branches": ["main"], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { "type": "feat", "release": "minor" }, + { "type": "fix", "release": "patch" }, + { "breaking": true, "release": "major" } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits" + } + ], + "@semantic-release/github" + ] +} +``` + +## PR Title Validation + +On pull requests, the pipeline validates that the PR title follows the [Conventional Commits](https://www.conventionalcommits.org/) format using [amannn/action-semantic-pull-request](https://github.com/amannn/action-semantic-pull-request). This is important because squash merges use the PR title as the commit message, which semantic-release uses to determine the next version. + +Valid examples: `feat: add user profiles`, `fix(auth): handle expired tokens`, `feat!: redesign API` + +## FAQ + +### Why is `@semantic-release/npm` excluded? +The default semantic-release config includes the npm plugin, which requires a `package.json` at the root. For non-Node.js projects, this causes an `ENOPKG` error. The `.releaserc.json` must explicitly list only the plugins you need. + +### How are images tagged? +On every run: ``. On PRs: also `pr-`. On main with a new release: also `` and `latest`. + +### Can I use this in a monorepo? +Yes. Set `service_path` to the subdirectory and add a `tagFormat` to your `.releaserc.json` (e.g., `"tagFormat": "my-service-v${version}"`) to scope releases per service. diff --git a/tests/service-pipeline/Dockerfile b/tests/service-pipeline/Dockerfile new file mode 100644 index 00000000..2be77aee --- /dev/null +++ b/tests/service-pipeline/Dockerfile @@ -0,0 +1,7 @@ +FROM alpine + +WORKDIR /app + +RUN echo "service-pipeline test" > hello.txt + +CMD ["cat", "hello.txt"] diff --git a/tests/service-pipeline/matrix-config.yaml b/tests/service-pipeline/matrix-config.yaml new file mode 100644 index 00000000..fc4d811f --- /dev/null +++ b/tests/service-pipeline/matrix-config.yaml @@ -0,0 +1,8 @@ +environments: + - test + +config: + test: + aws_account_id: "123456789012" + aws_region: eu-central-1 + aws_iam_role_name: test-role diff --git a/tests/service-pipeline/secrets.Dockerfile b/tests/service-pipeline/secrets.Dockerfile new file mode 100644 index 00000000..cf13cf67 --- /dev/null +++ b/tests/service-pipeline/secrets.Dockerfile @@ -0,0 +1,12 @@ +FROM alpine + +ARG CUSTOM_ARG=default + +WORKDIR /app + +RUN --mount=type=secret,id=SECRET_TOKEN \ + cp /run/secrets/SECRET_TOKEN secret.txt + +RUN echo "${CUSTOM_ARG}" > build-arg.txt + +CMD ["sh", "-c", "cat secret.txt && cat build-arg.txt"] From ceef37e1c653ce64f854082e6976a21581c50046 Mon Sep 17 00:00:00 2001 From: Samuel <15628653+swibrow@users.noreply.github.com> Date: Thu, 5 Feb 2026 14:04:23 +0100 Subject: [PATCH 2/2] feat: improve the gitops image tag to allow for regex searching --- .github/workflows/gitops-image-tag.yaml | 97 +++++-- docs/workflows/gitops-image-tag.md | 321 ++++++++++-------------- 2 files changed, 197 insertions(+), 221 deletions(-) diff --git a/.github/workflows/gitops-image-tag.yaml b/.github/workflows/gitops-image-tag.yaml index 4fd2a70a..0ff23729 100644 --- a/.github/workflows/gitops-image-tag.yaml +++ b/.github/workflows/gitops-image-tag.yaml @@ -11,9 +11,13 @@ on: type: string required: true image_tag_keys: - description: 'Newline-separated list of keys to update (e.g., webapp.image.tag)' + description: 'Newline-separated list of keys to update (e.g., webapp.image.tag). Ignored when image_name is set.' type: string - default: 'webapp.image_tag' + default: '' + image_name: + description: 'Image name to search for (e.g., social-media). Finds all image blocks where the repository ends with this name and updates their tag. Takes precedence over image_tag_keys.' + type: string + default: '' values_files: description: 'Newline-separated list of file paths to update' type: string @@ -58,8 +62,11 @@ jobs: - name: Update Helm values files run: | + IMAGE_TAG="${{ inputs.image_tag }}" + IMAGE_NAME="${{ inputs.image_name }}" + # Validate inputs - if [ -z "${{ inputs.image_tag }}" ]; then + if [ -z "$IMAGE_TAG" ]; then echo "Error: image_tag input cannot be empty" exit 1 fi @@ -67,12 +74,22 @@ jobs: echo "Error: values_files input cannot be empty" exit 1 fi + if [ -z "$IMAGE_NAME" ] && [ -z "${{ inputs.image_tag_keys }}" ]; then + echo "Error: either image_name or image_tag_keys must be provided" + exit 1 + fi echo "Files to update:" echo "${{ inputs.values_files }}" - echo "" - echo "Keys to update:" - echo "${{ inputs.image_tag_keys }}" + + if [ -n "$IMAGE_NAME" ]; then + echo "" + echo "Image name to search for: $IMAGE_NAME" + else + echo "" + echo "Keys to update:" + echo "${{ inputs.image_tag_keys }}" + fi # Process each file (newline-separated) while IFS= read -r file; do @@ -80,28 +97,52 @@ jobs: [ -z "$file" ] && continue echo "" - echo "Updating $file with image tag: ${{ inputs.image_tag }}" - - # Update each key in the file - while IFS= read -r key; do - [ -z "$key" ] && continue - - # Check if key exists before updating - if yq e ".$key" "$file" >/dev/null 2>&1 && [ "$(yq e ".$key" "$file")" != "null" ]; then - echo " Setting $key = ${{ inputs.image_tag }}" - # Get current value to check if update is needed - current_value=$(yq e ".$key" "$file") - if [ "$current_value" != "${{ inputs.image_tag }}" ]; then - # Use sed to replace the value while preserving formatting - sed -i "s|${key##*.}: .*|${key##*.}: ${{ inputs.image_tag }}|" "$file" + echo "Updating $file with image tag: $IMAGE_TAG" + + if [ -n "$IMAGE_NAME" ]; then + # --- IMAGE NAME SEARCH MODE --- + # Find all dot-notation paths to tag fields where sibling repository ends with the image name + TAG_PATHS=$(yq ' + (.. | select(has("repository") and has("tag")) | select(.repository | test("'"${IMAGE_NAME}"'$"))).tag | path | "." + join(".") + ' "$file" 2>/dev/null) + + if [ -z "$TAG_PATHS" ]; then + echo " No image blocks matching '$IMAGE_NAME' found in $file, skipping" + continue + fi + + while IFS= read -r tag_path; do + [ -z "$tag_path" ] && continue + current_value=$(yq "$tag_path" "$file") + if [ "$current_value" = "$IMAGE_TAG" ]; then + echo " $tag_path already up to date" + continue + fi + LINE=$(yq "$tag_path | line" "$file") + sed -i "${LINE}s|tag: .*|tag: \"${IMAGE_TAG}\"|" "$file" + echo " $tag_path = $IMAGE_TAG (line $LINE)" + done <<< "$TAG_PATHS" + else + # --- EXPLICIT KEYS MODE --- + while IFS= read -r key; do + [ -z "$key" ] && continue + + # Check if key exists before updating + if yq e ".$key" "$file" >/dev/null 2>&1 && [ "$(yq e ".$key" "$file")" != "null" ]; then + echo " Setting $key = $IMAGE_TAG" + current_value=$(yq e ".$key" "$file") + if [ "$current_value" != "$IMAGE_TAG" ]; then + # Use yq to find the exact line, then sed to preserve formatting + LINE=$(yq ".$key | line" "$file") + sed -i "${LINE}s|${key##*.}: .*|${key##*.}: \"${IMAGE_TAG}\"|" "$file" + else + echo " Value already up to date" + fi else - echo " Value already up to date" + echo " Key $key not found in $file, skipping" fi - else - echo " Key $key not found in $file, skipping" - exit 1 - fi - done <<< "${{ inputs.image_tag_keys }}" + done <<< "${{ inputs.image_tag_keys }}" + fi done <<< "${{ inputs.values_files }}" - name: Generate PR details @@ -127,9 +168,11 @@ jobs: ${{ inputs.image_tag }} ``` + **Image**: `${{ inputs.image_name || 'N/A' }}` + **Keys to update**: ``` - ${{ inputs.image_tag_keys }} + ${{ inputs.image_tag_keys || 'auto-detected via image_name' }} ``` **Files to update**: diff --git a/docs/workflows/gitops-image-tag.md b/docs/workflows/gitops-image-tag.md index 0476b60d..4c153d5b 100644 --- a/docs/workflows/gitops-image-tag.md +++ b/docs/workflows/gitops-image-tag.md @@ -1,10 +1,15 @@ --- -title: Update Helm Image Tag in Values.yaml +title: GitOps Image Tag --- ## Description -This GitHub Actions workflow updates the image tag in specified yaml files. It can create a pull request with the changes or commit them directly to the target branch. The workflow is designed to be reusable and can handle multiple files and keys. +Updates image tags in Helm values files and optionally creates a pull request. Supports two modes: + +- **Image name search** (`image_name`) — Automatically finds all image blocks (objects with `repository` and `tag` fields) where the repository ends with the given name, and updates their tags. This is the recommended approach as it requires no maintenance when adding new image references (e.g., cronjobs). +- **Explicit keys** (`image_tag_keys`) — Updates specific YAML keys by dot-notation path. Use this when you need precise control over which fields are updated. + +Both modes use `yq` to locate the exact line number, then `sed` to replace only that line, preserving the original file formatting. ### Inputs @@ -12,7 +17,8 @@ This GitHub Actions workflow updates the image tag in specified yaml files. It c | name | description | type | required | default | | --- | --- | --- | --- | --- | | `image_tag` |

The tag of the Docker image

| `string` | `true` | `""` | -| `image_tag_keys` |

Newline-separated list of keys to update (e.g., webapp.image.tag)

| `string` | `false` | `webapp.image_tag` | +| `image_tag_keys` |

Newline-separated list of keys to update (e.g., webapp.image.tag). Ignored when image_name is set.

| `string` | `false` | `""` | +| `image_name` |

Image name to search for (e.g., social-media). Finds all image blocks where the repository ends with this name and updates their tag. Takes precedence over imagetagkeys.

| `string` | `false` | `""` | | `values_files` |

Newline-separated list of file paths to update

| `string` | `true` | `""` | | `create_pr` |

Create a pull request. If false, the changes will be committed directly to the target branch. Github Actions must have write permissions to the repository. If branch protection is enabled a Github App is required and is able to bypass branch protection rules.

| `boolean` | `false` | `true` | | `pr_message` |

Custom message for the pull request. Defaults to a standard message.

| `string` | `false` | `This PR updates the Helm values files to use the latest image tag.` | @@ -26,255 +32,182 @@ This GitHub Actions workflow updates the image tag in specified yaml files. It c - -### Usage - -```yaml -jobs: - job1: - uses: dnd-it/github-workflows/.github/workflows/gitops-image-tag.yaml@v2 - with: - image_tag: - # The tag of the Docker image - # - # Type: string - # Required: true - # Default: "" - - image_tag_keys: - # Newline-separated list of keys to update (e.g., webapp.image.tag) - # - # Type: string - # Required: false - # Default: webapp.image_tag - - values_files: - # Newline-separated list of file paths to update - # - # Type: string - # Required: true - # Default: "" - - create_pr: - # Create a pull request. If false, the changes will be committed directly to the target branch. - # Github Actions must have write permissions to the repository. If branch protection is enabled a Github App is required and is able to bypass branch protection rules. - # - # Type: boolean - # Required: false - # Default: true - - pr_message: - # Custom message for the pull request. Defaults to a standard message. - # - # Type: string - # Required: false - # Default: This PR updates the Helm values files to use the latest image tag. - - auto_merge: - # Enable auto-merge for the pull request. Only works if create_pr is true. - # - # Type: boolean - # Required: false - # Default: false - - branch_name_prefix: - # Prefix for the branch name. - # - # Type: string - # Required: false - # Default: helm-values - - target_branch: - # The target branch for the pull request. Defaults the default branch of the repository. - # - # Type: string - # Required: false - # Default: ${{ github.event.repository.default_branch }} - - app_id: - # GitHub App ID for generating a token. Required if using GitHub App authentication. - # - # Type: string - # Required: false - # Default: "" -``` - - ## Examples -### Single File Update +### Image name search (recommended) -```yaml -name: Update Image Tag -on: - workflow_run: - workflows: ["Build and Push Docker Image"] - types: - - completed - -permissions: - contents: write - pull-requests: write +Automatically finds and updates all image tags matching the given name across base and environment-specific values files. The `image_name` uses a regex match against the `repository` field — it matches any repository ending with the given name. +```yaml jobs: update-image: - uses: dnd-it/github-workflows/.github/workflows/gitops-image-tag.yaml@v3 + uses: DND-IT/github-workflows/.github/workflows/gitops-image-tag.yaml@v3 with: - image_tag: ${{ github.sha }} - values_files: deploy/app/values.yaml - image_tag_keys: image_tag + image_tag: "1.2.0" + image_name: social-media + values_files: | + deploy/app/values.yaml + deploy/app/envs/dev/values.yaml + deploy/app/envs/prod/values.yaml create_pr: true + auto_merge: true + branch_name_prefix: helm/social-media + app_id: ${{ vars.GH_APP_ID }} + secrets: + app_private_key: ${{ secrets.GH_APP_PRIVATE_KEY }} ``` -### Multiple Files Update +Given a values file like: ```yaml -name: Update Image Tag in Multiple Environments -on: - workflow_dispatch: - inputs: - image_tag: - description: 'Docker image tag to deploy' - required: true - -permissions: - contents: write - pull-requests: write +generic: + image: + repository: social-media + tag: "1.0.0" # <-- updated + + cronjobs: + rss-ingestion: + image: + repository: 123456.dkr.ecr.eu-central-1.amazonaws.com/social-media + tag: "1.0.0" # <-- updated + +frontend: + image: + repository: social-media-frontend + tag: "2.0.0" # <-- NOT updated (doesn't end with "social-media") +``` +`image_name: social-media` matches any `repository` ending with `social-media` (including registry-prefixed variants like `123456.dkr.ecr.../social-media`) but does **not** match `social-media-frontend`. + +### With service-pipeline + +Typical usage after a service-pipeline build and release: + +```yaml jobs: - update-all-environments: - uses: dnd-it/github-workflows/.github/workflows/gitops-image-tag.yaml@v3 + pipeline: + uses: DND-IT/github-workflows/.github/workflows/service-pipeline.yaml@service-pipeline with: - image_tag: ${{ inputs.image_tag }} + service_name: social-media + service_path: backend + config_path: '.github/matrix-config.yaml' + app_id: ${{ vars.GH_APP_ID }} + secrets: + app_private_key: ${{ secrets.GH_APP_PRIVATE_KEY }} + + update-helm-values: + needs: pipeline + if: needs.pipeline.outputs.new_release_published == 'true' + permissions: + contents: write + pull-requests: write + uses: DND-IT/github-workflows/.github/workflows/gitops-image-tag.yaml@v3 + with: + image_tag: ${{ needs.pipeline.outputs.new_release_version }} + image_name: social-media values_files: | - deploy/dev/values.yaml - deploy/staging/values.yaml - deploy/prod/values.yaml - image_tag_keys: "app.image.tag" + deploy/app/values.yaml + deploy/app/envs/dev/values.yaml + deploy/app/envs/prod/values.yaml create_pr: true + auto_merge: true + branch_name_prefix: helm/social-media + app_id: ${{ vars.GH_APP_ID }} + secrets: + app_private_key: ${{ secrets.GH_APP_PRIVATE_KEY }} ``` -This creates a single PR that updates the image tag across all environment files, making it easy to promote a specific version across environments. +### Explicit keys -### Multiple Keys Update +Use when you need precise control over which fields to update: ```yaml -name: Update Multiple Image Tags -on: - workflow_dispatch: - inputs: - image_tag: - description: 'Docker image tag' - required: true - -permissions: - contents: write - pull-requests: write - jobs: - update-multiple-keys: - uses: dnd-it/github-workflows/.github/workflows/gitops-image-tag.yaml@v3 + update-image: + uses: DND-IT/github-workflows/.github/workflows/gitops-image-tag.yaml@v3 with: - image_tag: ${{ inputs.image_tag }} + image_tag: "1.2.0" image_tag_keys: | - webapp.image.tag - worker.image.tag - api.image.tag + generic.image.tag + generic.cronjobs.draft-cleanup.image.tag values_files: deploy/app/values.yaml create_pr: true ``` -This updates multiple services that share the same Docker image. - -### Auto-merge Example +### Multiple files ```yaml -name: Deploy to Production -on: - release: - types: [published] +jobs: + update-all-environments: + uses: DND-IT/github-workflows/.github/workflows/gitops-image-tag.yaml@v3 + with: + image_tag: ${{ inputs.image_tag }} + image_tag_keys: app.image.tag + values_files: | + deploy/dev/values.yaml + deploy/staging/values.yaml + deploy/prod/values.yaml + create_pr: true +``` -permissions: - contents: write - pull-requests: write +### Auto-merge +```yaml jobs: deploy: - uses: dnd-it/github-workflows/.github/workflows/gitops-image-tag.yaml@v3 + uses: DND-IT/github-workflows/.github/workflows/gitops-image-tag.yaml@v3 with: image_tag: ${{ github.event.release.tag_name }} - values_files: | - deploy/prod/values.yaml - deploy/prod/secondary/values.yaml - image_tag_keys: | - app.image.tag - worker.image.tag + image_name: my-service + values_files: deploy/prod/values.yaml create_pr: true auto_merge: true - pr_message: | - Automated deployment of release ${{ github.event.release.tag_name }} to production. - - Release notes: ${{ github.event.release.html_url }} + app_id: ${{ vars.GH_APP_ID }} + secrets: + app_private_key: ${{ secrets.GH_APP_PRIVATE_KEY }} ``` -This creates a PR that will automatically merge once all checks pass, providing an audit trail while enabling continuous deployment. - -## Key Features +## How it works -### Automatic PR Replacement +1. For each file in `values_files`: + - **Image name mode**: Uses `yq` to recursively find all objects with `repository` and `tag` fields where `repository` ends with the given `image_name`. Extracts the dot-notation path to each `tag` field. + - **Explicit keys mode**: Uses each key from `image_tag_keys` directly. +2. For each tag field found, gets the exact line number via `yq ... | line`. +3. Uses `sed` on that specific line to replace the value, preserving indentation and formatting. +4. Creates a PR (or commits directly) with the changes. -The workflow now generates consistent branch names based on the files being updated. This means: -- If you update the same files again, the new PR will replace the old one -- No more accumulating duplicate PRs for the same files -- Branch names are based on a hash of the files being updated +## Key Features -### Multiple Files Support +### Image name search -You can update multiple values files in a single workflow run: -- Use the `values_files` input with newline-separated file paths -- All files are updated in a single commit/PR +The `image_name` input automatically discovers all image references in a values file, including deeply nested ones like cronjob images. No need to manually list every key path — add a new cronjob with the same image and it gets picked up automatically. -### Multiple Keys Support +### Automatic PR replacement -You can update multiple keys with the same image tag: -- Use the `image_tag_keys` input with newline-separated key paths -- Useful when multiple services share the same image +The workflow generates consistent branch names based on the files being updated. If you update the same files again, the new PR will replace the old one. No accumulating duplicate PRs. -### Enhanced PR Management +### Formatting preservation -- Detailed PR descriptions showing all updated files -- Option to commit directly without creating a PR +Unlike a pure `yq` write which can reformat the entire file, this workflow uses `yq` only to locate the target line and `sed` to perform the replacement. This preserves comments, indentation, and quoting style. ## FAQ -**Q: How can I update multiple values files?** +**Q: When should I use `image_name` vs `image_tag_keys`?** -A: Use the `values_files` input with newline-separated file paths: -```yaml -values_files: | - file1.yaml - file2.yaml - file3.yaml -``` +A: Use `image_name` (recommended) when all images sharing a name should have the same tag. Use `image_tag_keys` when you need to update specific keys that don't follow the `repository`/`tag` pattern, or when different images with similar names need different tags. -**Q: Will this create multiple PRs if I run it multiple times?** +**Q: How does `image_name` matching work?** -A: No, the workflow uses consistent branch naming based on the files being updated. Running it again will update the existing PR. +A: It uses a regex `test("name$")` — matching any `repository` value that **ends with** the given name. This handles both plain names (`social-media`) and registry-prefixed names (`123456.dkr.ecr.eu-central-1.amazonaws.com/social-media`). -**Q: Can I update multiple keys with the same image tag?** +**Q: What happens if a key doesn't exist in a file?** -A: Yes, use the `image_tag_keys` input with newline-separated key paths: -```yaml -image_tag_keys: | - frontend.image.tag - backend.image.tag - worker.image.tag -``` +A: In explicit keys mode, the workflow skips that key and logs a message. In image name mode, if no matching image blocks are found in a file, it skips the entire file. -**Q: What happens if a key doesn't exist in a file?** +**Q: Will this create multiple PRs if I run it multiple times?** -A: The workflow will skip that key and log a message. It only updates existing keys to prevent accidentally creating new structures. +A: No, the workflow uses consistent branch naming based on the files being updated. Running it again will update the existing PR. -**Q: Is the values_files input required?** +**Q: Can I use both `image_name` and `image_tag_keys`?** -A: Yes, the `values_files` input is required. You must specify at least one file path to update. +A: If both are provided, `image_name` takes precedence and `image_tag_keys` is ignored.