Skip to content
This repository was archived by the owner on Nov 17, 2025. It is now read-only.

Commit 00c6c7e

Browse files
yonchyonch
authored
feature: add dependency-aware build (#34)
* add libbpf * also compile bpftool * try building on ubuntu-24.04 * add input to control registry location of the built image * add parallel build to gha * add checkout to be able to refer to action * simplify getting submodules * add registry to image tag so build stages can find their dependencies * calculate recursive dependencies * fix output of hash computation function * fix transfering the image tags * dry run only affects final push * add force_rebuild parameter so we can ensure we get new versions from package repositories * remove old workflow * change new workflow's description
1 parent 8da381a commit 00c6c7e

File tree

9 files changed

+755
-108
lines changed

9 files changed

+755
-108
lines changed

‎.github/actions/build-push-container/action.yml‎

Lines changed: 275 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,275 @@
1+
# Copyright The OpenTelemetry Authors
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
name: 'Build and Push Container'
5+
description: 'Build and push a Docker container with dependency management and registry caching'
6+
7+
inputs:
8+
directory:
9+
description: 'Directory name to build'
10+
required: true
11+
registry:
12+
description: 'Container registry to use'
13+
required: true
14+
default: 'ghcr.io'
15+
registry_username:
16+
description: 'Registry username'
17+
required: true
18+
registry_password:
19+
description: 'Registry password/token'
20+
required: true
21+
image_prefix:
22+
description: 'Prefix for image names'
23+
required: false
24+
default: 'benv'
25+
ref:
26+
description: 'Git ref to checkout'
27+
required: false
28+
default: 'main'
29+
force_rebuild:
30+
description: 'Force rebuild and push even if image exists in registry'
31+
required: false
32+
default: 'false'
33+
34+
outputs:
35+
image-tag:
36+
description: 'The computed image tag'
37+
value: ${{ steps.compute-recursive-tags.outputs.image-tag }}
38+
full-image-tag:
39+
description: 'The full image tag with registry'
40+
value: ${{ steps.compute-recursive-tags.outputs.full-image-tag }}
41+
image-exists:
42+
description: 'Whether the image already exists in registry'
43+
value: ${{ steps.check-exists.outputs.exists }}
44+
build-needed:
45+
description: 'Whether a build was needed'
46+
value: ${{ steps.check-exists.outputs.exists == 'false' }}
47+
48+
runs:
49+
using: 'composite'
50+
steps:
51+
- name: Checkout sources
52+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
53+
with:
54+
ref: ${{ inputs.ref }}
55+
fetch-depth: 0
56+
57+
- name: Initialize directory-specific submodules
58+
shell: bash
59+
run: |
60+
DIRECTORY="${{ inputs.directory }}"
61+
echo "Initializing submodules for directory: ${DIRECTORY}"
62+
63+
# Initialize submodules for the specific directory path
64+
git submodule update --init --recursive -- "${DIRECTORY}/"
65+
66+
- name: Compute recursive tags for all directories
67+
id: compute-recursive-tags
68+
shell: bash
69+
env:
70+
DOCKER_TAG_PREFIX: ${{ github.repository_owner }}/
71+
run: |
72+
DIRECTORY="${{ inputs.directory }}"
73+
74+
# Define dependency mapping based on CMakeLists.txt
75+
declare -A DEPS
76+
DEPS["base"]=""
77+
DEPS["openssl"]="base"
78+
DEPS["bcc"]="base"
79+
DEPS["libuv"]="base"
80+
DEPS["cpp_misc"]="base"
81+
DEPS["go"]="base"
82+
DEPS["abseil_cpp"]="base"
83+
DEPS["libmaxminddb"]="base"
84+
DEPS["libbpf"]="base"
85+
DEPS["curl"]="base openssl"
86+
DEPS["grpc_cpp"]="base abseil_cpp openssl"
87+
DEPS["aws_sdk"]="base openssl curl"
88+
DEPS["gcp_cpp"]="base openssl curl grpc_cpp"
89+
DEPS["opentelemetry"]="base grpc_cpp"
90+
DEPS["final"]="base openssl curl bcc libuv aws_sdk cpp_misc go grpc_cpp abseil_cpp libmaxminddb gcp_cpp opentelemetry libbpf"
91+
92+
# Compute direct hashes for all directories upfront
93+
declare -A DIRECT_HASHES
94+
ALL_DIRS="base openssl bcc libuv cpp_misc go abseil_cpp libmaxminddb libbpf curl grpc_cpp aws_sdk gcp_cpp opentelemetry final"
95+
96+
echo "Computing direct hashes..." >&2
97+
for dir in $ALL_DIRS; do
98+
direct_hash=$(git log -1 --format=%h "${dir}")
99+
DIRECT_HASHES[$dir]=$direct_hash
100+
echo "Direct hash for $dir: $direct_hash" >&2
101+
done
102+
103+
# Function to compute dependency closure (all transitive dependencies)
104+
compute_closure() {
105+
local target="$1"
106+
local visited_key="VISITED_$target"
107+
108+
# Check for circular dependency
109+
if [[ -n "${!visited_key:-}" ]]; then
110+
echo "ERROR: Circular dependency detected for $target" >&2
111+
exit 1
112+
fi
113+
114+
# Mark as visiting
115+
declare -g "$visited_key=1"
116+
117+
# Start with direct dependencies
118+
local deps="${DEPS[$target]:-}"
119+
local closure_set=""
120+
121+
# Add direct dependencies
122+
for dep in $deps; do
123+
closure_set="$closure_set $dep"
124+
125+
# Recursively add their closures
126+
local dep_closure=$(compute_closure "$dep")
127+
closure_set="$closure_set $dep_closure"
128+
done
129+
130+
# Remove duplicates by converting to array and back
131+
local unique_closure=($(echo $closure_set | tr ' ' '\n' | sort -u | tr '\n' ' '))
132+
133+
# Unmark visiting
134+
unset "$visited_key"
135+
136+
echo "${unique_closure[@]}"
137+
}
138+
139+
# Function to compute recursive hash using closure approach
140+
compute_recursive_hash() {
141+
local dir="$1"
142+
143+
# Get the full dependency closure
144+
local closure=$(compute_closure "$dir")
145+
146+
# Include the directory itself in the hash computation
147+
local all_dirs_for_hash="$dir $closure"
148+
149+
# Sort all directories
150+
local sorted_dirs=($(echo $all_dirs_for_hash | tr ' ' '\n' | sort -u | tr '\n' ' '))
151+
152+
# Concatenate their direct hashes with dashes
153+
local hash_input=""
154+
for d in "${sorted_dirs[@]}"; do
155+
if [[ -n "$d" ]]; then
156+
if [[ -n "$hash_input" ]]; then
157+
hash_input="$hash_input-${DIRECT_HASHES[$d]}"
158+
else
159+
hash_input="${DIRECT_HASHES[$d]}"
160+
fi
161+
fi
162+
done
163+
164+
# Use the dash-separated hashes directly as the tag
165+
local final_hash="$hash_input"
166+
167+
echo "Closure for $dir: ${sorted_dirs[@]}" >&2
168+
echo "Final hash for $dir: $final_hash" >&2
169+
170+
echo "$final_hash"
171+
}
172+
173+
# Compute recursive hash for target directory
174+
RECURSIVE_HASH=$(compute_recursive_hash "$DIRECTORY")
175+
176+
# Create image tag
177+
IMAGE_TAG="${{ github.repository_owner }}/benv-${DIRECTORY}:${RECURSIVE_HASH}"
178+
FULL_IMAGE_TAG="${{ inputs.registry }}/${IMAGE_TAG}"
179+
180+
echo "image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT
181+
echo "full-image-tag=${FULL_IMAGE_TAG}" >> $GITHUB_OUTPUT
182+
echo "recursive-hash=${RECURSIVE_HASH}" >> $GITHUB_OUTPUT
183+
184+
echo "Computed recursive image tag: ${IMAGE_TAG}" >&2
185+
echo "Full image tag: ${FULL_IMAGE_TAG}" >&2
186+
echo "Recursive hash: ${RECURSIVE_HASH}" >&2
187+
188+
# Compute all dependency tags for build args
189+
echo "Computing all dependency tags..." >&2
190+
ALL_DIRS="base openssl bcc libuv cpp_misc go abseil_cpp libmaxminddb libbpf curl grpc_cpp aws_sdk gcp_cpp opentelemetry final"
191+
192+
for dir in $ALL_DIRS; do
193+
if [[ "$dir" != "$DIRECTORY" ]]; then
194+
dir_hash=$(compute_recursive_hash "$dir")
195+
dir_image_tag="${{ github.repository_owner }}/benv-${dir}:${dir_hash}"
196+
dir_full_tag="${{ inputs.registry }}/${dir_image_tag}"
197+
198+
# Export as environment variable for use in build args
199+
export "${dir}_IMAGE_TAG=${dir_full_tag}"
200+
echo "${dir}_IMAGE_TAG=${dir_full_tag}" >> $GITHUB_OUTPUT
201+
202+
echo "Dependency: ${dir} -> ${dir_full_tag}" >&2
203+
fi
204+
done
205+
206+
- name: Check if image exists in registry
207+
id: check-exists
208+
shell: bash
209+
run: |
210+
FULL_IMAGE_TAG="${{ steps.compute-recursive-tags.outputs.full-image-tag }}"
211+
212+
if [[ "${{ inputs.force_rebuild }}" == "true" ]]; then
213+
echo "exists=false" >> $GITHUB_OUTPUT
214+
echo "Force rebuild enabled - will rebuild ${FULL_IMAGE_TAG} regardless of registry state"
215+
elif docker manifest inspect "${FULL_IMAGE_TAG}" >/dev/null 2>&1; then
216+
echo "exists=true" >> $GITHUB_OUTPUT
217+
echo "Image ${FULL_IMAGE_TAG} already exists in registry"
218+
else
219+
echo "exists=false" >> $GITHUB_OUTPUT
220+
echo "Image ${FULL_IMAGE_TAG} does not exist in registry"
221+
fi
222+
223+
- name: Log in to Container Registry
224+
if: steps.check-exists.outputs.exists == 'false'
225+
uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
226+
with:
227+
registry: ${{ inputs.registry }}
228+
username: ${{ inputs.registry_username }}
229+
password: ${{ inputs.registry_password }}
230+
231+
- name: Build and push image
232+
if: steps.check-exists.outputs.exists == 'false'
233+
shell: bash
234+
run: |
235+
DIRECTORY="${{ inputs.directory }}"
236+
FULL_IMAGE_TAG="${{ steps.compute-recursive-tags.outputs.full-image-tag }}"
237+
238+
# Start building the docker command
239+
BUILD_ARGS="--build-arg NPROC=$(nproc)"
240+
241+
# Add all dependency image tags as build args using outputs from compute-recursive-tags step
242+
BUILD_ARGS="${BUILD_ARGS} --build-arg base_IMAGE_TAG=${{ steps.compute-recursive-tags.outputs.base_IMAGE_TAG }}"
243+
BUILD_ARGS="${BUILD_ARGS} --build-arg openssl_IMAGE_TAG=${{ steps.compute-recursive-tags.outputs.openssl_IMAGE_TAG }}"
244+
BUILD_ARGS="${BUILD_ARGS} --build-arg bcc_IMAGE_TAG=${{ steps.compute-recursive-tags.outputs.bcc_IMAGE_TAG }}"
245+
BUILD_ARGS="${BUILD_ARGS} --build-arg libuv_IMAGE_TAG=${{ steps.compute-recursive-tags.outputs.libuv_IMAGE_TAG }}"
246+
BUILD_ARGS="${BUILD_ARGS} --build-arg cpp_misc_IMAGE_TAG=${{ steps.compute-recursive-tags.outputs.cpp_misc_IMAGE_TAG }}"
247+
BUILD_ARGS="${BUILD_ARGS} --build-arg go_IMAGE_TAG=${{ steps.compute-recursive-tags.outputs.go_IMAGE_TAG }}"
248+
BUILD_ARGS="${BUILD_ARGS} --build-arg abseil_cpp_IMAGE_TAG=${{ steps.compute-recursive-tags.outputs.abseil_cpp_IMAGE_TAG }}"
249+
BUILD_ARGS="${BUILD_ARGS} --build-arg libmaxminddb_IMAGE_TAG=${{ steps.compute-recursive-tags.outputs.libmaxminddb_IMAGE_TAG }}"
250+
BUILD_ARGS="${BUILD_ARGS} --build-arg libbpf_IMAGE_TAG=${{ steps.compute-recursive-tags.outputs.libbpf_IMAGE_TAG }}"
251+
BUILD_ARGS="${BUILD_ARGS} --build-arg curl_IMAGE_TAG=${{ steps.compute-recursive-tags.outputs.curl_IMAGE_TAG }}"
252+
BUILD_ARGS="${BUILD_ARGS} --build-arg grpc_cpp_IMAGE_TAG=${{ steps.compute-recursive-tags.outputs.grpc_cpp_IMAGE_TAG }}"
253+
BUILD_ARGS="${BUILD_ARGS} --build-arg aws_sdk_IMAGE_TAG=${{ steps.compute-recursive-tags.outputs.aws_sdk_IMAGE_TAG }}"
254+
BUILD_ARGS="${BUILD_ARGS} --build-arg gcp_cpp_IMAGE_TAG=${{ steps.compute-recursive-tags.outputs.gcp_cpp_IMAGE_TAG }}"
255+
BUILD_ARGS="${BUILD_ARGS} --build-arg opentelemetry_IMAGE_TAG=${{ steps.compute-recursive-tags.outputs.opentelemetry_IMAGE_TAG }}"
256+
BUILD_ARGS="${BUILD_ARGS} --build-arg final_IMAGE_TAG=${{ steps.compute-recursive-tags.outputs.final_IMAGE_TAG }}"
257+
258+
# Add environment-specific build args if they exist
259+
if [ -n "${BENV_BASE_IMAGE_DISTRO}" ]; then
260+
BUILD_ARGS="${BUILD_ARGS} --build-arg BENV_BASE_IMAGE_DISTRO=${BENV_BASE_IMAGE_DISTRO}"
261+
fi
262+
263+
if [ -n "${BENV_BASE_IMAGE_VERSION}" ]; then
264+
BUILD_ARGS="${BUILD_ARGS} --build-arg BENV_BASE_IMAGE_VERSION=${BENV_BASE_IMAGE_VERSION}"
265+
fi
266+
267+
# Build the image
268+
echo "Building image: ${FULL_IMAGE_TAG}"
269+
echo "Build args: ${BUILD_ARGS}"
270+
271+
docker build -t "${FULL_IMAGE_TAG}" ${BUILD_ARGS} "${DIRECTORY}/"
272+
273+
# Always push intermediate builds to cache registry (dry_run only affects final Docker Hub push)
274+
echo "Pushing image to cache registry: ${FULL_IMAGE_TAG}"
275+
docker push "${FULL_IMAGE_TAG}"

‎.github/workflows/build_and_push.yml‎

Lines changed: 0 additions & 108 deletions
This file was deleted.

0 commit comments

Comments
 (0)