From 86de9ed1b6543f283c5006a87c7db3a7653e2aaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chojnacki?= Date: Mon, 20 Apr 2026 19:45:00 +0200 Subject: [PATCH 01/16] Add devcontainer workflow for local builds and integration tests Introduce a .devcontainer image definition (moved from .gitlab/) and a thin Makefile + runner script so developers can reproduce the GitLab build + integration-test pipeline locally. The image bakes in the LLVM+musl sysroot, Rust+cbindgen for RUM, uv for Python test setup, and a prebuilt httpd 2.4. Notable details in the runner script: - UV_PROJECT_ENVIRONMENT is pointed outside the bind-mounted repo so a host-side `uv sync` on darwin can't leave mac wheels sitting in .venv/ that the container's Linux Python then tries to import. - Pytest options are passed with `--flag=VALUE` (single token). The space-separated form gets captured by pytest's early arg parser as a positional test path before conftest registers the custom option, which makes pytest use the file's parent as rootdir and silently skip conftest. The GitLab build-ci-image job's hash input and Dockerfile path are retargeted to the new .devcontainer/ location. --- {.gitlab => .devcontainer}/Dockerfile | 0 .devcontainer/devcontainer.json | 10 +++ .devcontainer/devcontainer.mk | 104 +++++++++++++++++++++++++ .devcontainer/run-integration-tests.sh | 44 +++++++++++ .gitignore | 3 + .gitlab-ci.yml | 4 +- Makefile | 18 +---- 7 files changed, 164 insertions(+), 19 deletions(-) rename {.gitlab => .devcontainer}/Dockerfile (100%) create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/devcontainer.mk create mode 100755 .devcontainer/run-integration-tests.sh diff --git a/.gitlab/Dockerfile b/.devcontainer/Dockerfile similarity index 100% rename from .gitlab/Dockerfile rename to .devcontainer/Dockerfile diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..5c34a2a --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,10 @@ +{ + "name": "httpd-datadog", + "build": { + "dockerfile": "Dockerfile", + "context": "..", + "args": { + "ARCH": "${localEnv:ARCH:x86_64}" + } + } +} diff --git a/.devcontainer/devcontainer.mk b/.devcontainer/devcontainer.mk new file mode 100644 index 0000000..a167eb1 --- /dev/null +++ b/.devcontainer/devcontainer.mk @@ -0,0 +1,104 @@ +DEV_CONTAINER_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) +DEV_CONTAINER_REPO_ROOT := $(abspath $(DEV_CONTAINER_DIR)/..) + +# When running inside the container, tools are available directly. +# Check for /.dockerenv (Docker) or KUBERNETES_SERVICE_HOST (Kubernetes CI). +ifneq ($(or $(wildcard /.dockerenv),$(KUBERNETES_SERVICE_HOST)),) + +DEVCONTAINER_RUN := + +.PHONY: dev-image +dev-image: + @true + +else + +DEV_CONTAINER_REGISTRY ?= registry.ddbuild.io +DEV_CONTAINER_IMAGE_NAME ?= $(DEV_CONTAINER_REGISTRY)/ci/httpd-datadog + +# Match the hash computation in .gitlab-ci.yml (build-ci-image job): sha256 over +# .devcontainer/Dockerfile, deps/nginx-datadog/build_env/, and scripts/setup-httpd.py. +DEV_CONTAINER_HASH := $(shell cd $(DEV_CONTAINER_REPO_ROOT) && \ + find .devcontainer/Dockerfile deps/nginx-datadog/build_env/ scripts/setup-httpd.py -type f | sort | \ + while IFS= read -r f; do echo "--- $$f ---"; cat "$$f"; done | \ + (command -v sha256sum >/dev/null 2>&1 && sha256sum || shasum -a 256) | cut -d' ' -f1) + +DEV_CONTAINER_TAG := $(DEV_CONTAINER_HASH) +DEV_CONTAINER_IMAGE := $(DEV_CONTAINER_IMAGE_NAME):$(DEV_CONTAINER_TAG) + +# Map host arch (uname -m) to the Dockerfile's ARCH build-arg. +DEV_CONTAINER_HOST_ARCH := $(shell uname -m) +ifeq ($(DEV_CONTAINER_HOST_ARCH),arm64) +DEV_CONTAINER_ARCH := aarch64 +DEV_CONTAINER_PLATFORM := linux/arm64 +else ifeq ($(DEV_CONTAINER_HOST_ARCH),aarch64) +DEV_CONTAINER_ARCH := aarch64 +DEV_CONTAINER_PLATFORM := linux/arm64 +else +DEV_CONTAINER_ARCH := x86_64 +DEV_CONTAINER_PLATFORM := linux/amd64 +endif + +# For git worktrees, .git is a file pointing to a gitdir outside the repo root; +# bind-mount the git common-dir so `git` works inside the container. +DEV_CONTAINER_GIT_COMMON_DIR := $(shell cd $(DEV_CONTAINER_REPO_ROOT) && git rev-parse --path-format=absolute --git-common-dir 2>/dev/null) +ifneq ($(DEV_CONTAINER_GIT_COMMON_DIR),) +ifneq ($(DEV_CONTAINER_GIT_COMMON_DIR),$(DEV_CONTAINER_REPO_ROOT)/.git) +DEV_CONTAINER_GIT_MOUNT := -v $(DEV_CONTAINER_GIT_COMMON_DIR):$(DEV_CONTAINER_GIT_COMMON_DIR) +endif +endif + +DEVCONTAINER_RUN = docker run --rm \ + -v $(DEV_CONTAINER_REPO_ROOT):$(DEV_CONTAINER_REPO_ROOT) \ + $(DEV_CONTAINER_GIT_MOUNT) \ + -w $(DEV_CONTAINER_REPO_ROOT) \ + $(DEV_CONTAINER_IMAGE) + +# Ensure the dev container image is available locally. Prefer pulling from the +# registry (CI builds every tagged hash); fall back to a local build. +# Stale images with different tags are removed first. +.PHONY: dev-image +dev-image: + @if ! docker image inspect $(DEV_CONTAINER_IMAGE) >/dev/null 2>&1; then \ + stale=$$(docker images --format '{{.Repository}}:{{.Tag}}' $(DEV_CONTAINER_IMAGE_NAME) 2>/dev/null | grep -v ':$(DEV_CONTAINER_TAG)$$' || true); \ + if [ -n "$$stale" ]; then \ + echo "Removing stale dev container image(s): $$stale"; \ + docker rmi $$stale || true; \ + fi; \ + if docker pull $(DEV_CONTAINER_IMAGE) >/dev/null 2>&1; then \ + echo "Pulled $(DEV_CONTAINER_IMAGE)"; \ + else \ + echo "Building dev container image $(DEV_CONTAINER_IMAGE)..."; \ + docker buildx build \ + --platform $(DEV_CONTAINER_PLATFORM) \ + --build-arg ARCH=$(DEV_CONTAINER_ARCH) \ + --load \ + -t $(DEV_CONTAINER_IMAGE) \ + -f $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/Dockerfile \ + $(DEV_CONTAINER_REPO_ROOT); \ + fi; \ + fi + +.PHONY: dev-shell +dev-shell: dev-image + docker run --rm -it \ + -v $(DEV_CONTAINER_REPO_ROOT):$(DEV_CONTAINER_REPO_ROOT) \ + -w $(DEV_CONTAINER_REPO_ROOT) \ + $(DEV_CONTAINER_IMAGE) \ + /bin/sh + +.PHONY: test-integration +test-integration: dev-image + $(DEVCONTAINER_RUN) .devcontainer/run-integration-tests.sh + +.PHONY: dev-image-clean +dev-image-clean: + @stale=$$(docker images --format '{{.Repository}}:{{.Tag}}' $(DEV_CONTAINER_IMAGE_NAME) 2>/dev/null | grep -v ':$(DEV_CONTAINER_TAG)$$' || true); \ + if [ -n "$$stale" ]; then \ + echo "Removing stale dev container image(s): $$stale"; \ + docker rmi $$stale; \ + else \ + echo "No stale dev container images found."; \ + fi + +endif diff --git a/.devcontainer/run-integration-tests.sh b/.devcontainer/run-integration-tests.sh new file mode 100755 index 0000000..1c1c10b --- /dev/null +++ b/.devcontainer/run-integration-tests.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# Build mod_datadog (with RUM) and run the full integration-test suite. +# Assumes it runs inside the devcontainer image (Alpine + musl sysroot + httpd +# at /httpd/httpd-build/ + uv). +set -eux + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$REPO_ROOT" + +ARCH="$(uname -m)" + +# Use a container-specific build tree to avoid colliding with host builds +# (host cmake would use Unix Makefiles; the ci-release preset uses Ninja). +BUILD_DIR="$REPO_ROOT/build-container" +DIST_DIR="$REPO_ROOT/dist-container" + +# Repo root is bind-mounted from the host; cmake complains otherwise. +git config --global --add safe.directory "$REPO_ROOT" + +cmake --preset=ci-release \ + -DHTTPD_DATADOG_ENABLE_RUM=ON \ + -DCMAKE_TOOLCHAIN_FILE="/sysroot/${ARCH}-none-linux-musl/Toolchain.cmake" \ + -B "$BUILD_DIR" . +cmake --build "$BUILD_DIR" -j +cmake --install "$BUILD_DIR" --prefix "$DIST_DIR" + +cd test/integration-test + +# Keep the venv outside the bind-mounted repo so a host-side `uv sync` +# (darwin/arm64 wheels) can't shadow the container's Linux one. +export UV_PROJECT_ENVIRONMENT=/root/.venv-httpd-datadog-tests + +uv sync + +# Pytest's early arg parser runs BEFORE conftest registers `--bin-path`. +# If we pass `--bin-path VALUE` (space-separated), that parser treats the +# value as a positional test path and uses its parent as rootdir, which +# then prevents conftest from loading. The `=` syntax keeps the option and +# its value in a single token and avoids that race. +uv run pytest \ + --bin-path=/httpd/httpd-build/bin/apachectl \ + --module-path="$DIST_DIR/lib/mod_datadog.so" \ + --log-dir="$REPO_ROOT/logs" \ + -v "$@" diff --git a/.gitignore b/.gitignore index 75feb22..92a9c5e 100644 --- a/.gitignore +++ b/.gitignore @@ -23,10 +23,13 @@ compile_commands.json __pycache__/ +build-container/ build-rum/ build/ +dist-container/ httpd-*/ httpd/ +/logs/ venv/ test/integration-test/.coverage diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4a328df..b7f8077 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,7 +58,7 @@ build-ci-image: TOOLCHAIN_ARCH: aarch64 script: - | - FILES=$(find .gitlab/Dockerfile deps/nginx-datadog/build_env/ scripts/setup-httpd.py -type f | sort) + FILES=$(find .devcontainer/Dockerfile deps/nginx-datadog/build_env/ scripts/setup-httpd.py -type f | sort) HASH=$(for f in $FILES; do echo "--- $f ---"; cat "$f"; done | sha256sum | cut -d ' ' -f 1) - echo "CI_IMAGE_HASH=$HASH" >> ci-image.env - IMAGE_TAG="$CI_DOCKER_IMAGE_BASE/$ARCH:$HASH" @@ -71,7 +71,7 @@ build-ci-image: --platform linux/$ARCH \ --build-arg ARCH=$TOOLCHAIN_ARCH \ --push \ - --file .gitlab/Dockerfile \ + --file .devcontainer/Dockerfile \ . echo "Image $IMAGE_TAG built for $ARCH." fi diff --git a/Makefile b/Makefile index 4360495..3b5374c 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1 @@ -# The CI image used for some GitHub jobs is built by the GitLab build-ci-image job. -# The hash is computed by this GitLab job from the files used to build the image. -# -# Whenever this image needs to be updated, one should: -# - Get the hash from a run of the GitLab build-ci-image job. -# - Copy this hash here, and in the GitHub workflow files. -# - Run: make replicate-ci-image-for-github. - -CI_DOCKER_IMAGE_HASH ?= 28219c0ef3e00f1e3d5afcab61a73a5e9bd2a9b957d7545556711cce2a6262cd -CI_IMAGE_FROM_GITLAB ?= registry.ddbuild.io/ci/httpd-datadog/amd64:$(CI_DOCKER_IMAGE_HASH) -CI_IMAGE_IN_PUBLIC_REPO_FOR_GITHUB ?= datadog/docker-library:httpd-datadog-ci-$(CI_DOCKER_IMAGE_HASH) - -.PHONY: replicate-ci-image-for-github -replicate-ci-image-for-github: - docker pull $(CI_IMAGE_FROM_GITLAB) - docker tag $(CI_IMAGE_FROM_GITLAB) $(CI_IMAGE_IN_PUBLIC_REPO_FOR_GITHUB) - docker push $(CI_IMAGE_IN_PUBLIC_REPO_FOR_GITHUB) +include .devcontainer/devcontainer.mk From 2b9dc7c7674d353406298e86ed6437c0f7927cbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chojnacki?= Date: Mon, 20 Apr 2026 19:50:00 +0200 Subject: [PATCH 02/16] Consolidate devcontainer image build into one job Merge the three-job ci-image flow (build-ci-image + nydusify-ci-image + create-ci-manifest) into two: a single `devcontainer-image` matrix job that builds + pushes + nydusifies each arch inline, followed by `create-ci-manifest` which still stitches the per-arch nydus images into a multi-arch index. Drops the separate nydusify job entirely. Key changes: - Single job uses the NYDUS_DD_IMAGE bundle (buildx + crane + nydus-convert all in one) instead of the plain docker image, so the build-push-nydusify sequence happens in one runner. - Per-arch nydus conversion lands directly on $CI_DOCKER_IMAGE_BASE/$ARCH:$HASH before the manifest is assembled, so the final multi-arch manifest references nydus layers without a separate manifest-level conversion step. - Export DEVCONTAINER_IMAGE=$base:$HASH (full ref) via dotenv so downstream jobs can pin the image by a single variable and don't have to re-construct the ref from a base+hash pair. Also prepares run-integration-tests.sh for CI reuse: if MODULE_PATH is set in the environment, the cmake build is skipped and pytest consumes the pre-built module at that path. Default behavior (no envvar) is unchanged, so local dev still gets a full build-and-test. --- .devcontainer/run-integration-tests.sh | 22 ++++--- .gitlab-ci.yml | 85 ++++++++++++-------------- 2 files changed, 54 insertions(+), 53 deletions(-) diff --git a/.devcontainer/run-integration-tests.sh b/.devcontainer/run-integration-tests.sh index 1c1c10b..84579ac 100755 --- a/.devcontainer/run-integration-tests.sh +++ b/.devcontainer/run-integration-tests.sh @@ -2,6 +2,11 @@ # Build mod_datadog (with RUM) and run the full integration-test suite. # Assumes it runs inside the devcontainer image (Alpine + musl sysroot + httpd # at /httpd/httpd-build/ + uv). +# +# If MODULE_PATH is set in the environment, the cmake build is skipped and +# pytest is pointed at the pre-built module at that path. CI uses this to +# reuse the artifact produced by the build:$ARCH job instead of rebuilding +# from source inside the test job. set -eux REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" @@ -17,12 +22,15 @@ DIST_DIR="$REPO_ROOT/dist-container" # Repo root is bind-mounted from the host; cmake complains otherwise. git config --global --add safe.directory "$REPO_ROOT" -cmake --preset=ci-release \ - -DHTTPD_DATADOG_ENABLE_RUM=ON \ - -DCMAKE_TOOLCHAIN_FILE="/sysroot/${ARCH}-none-linux-musl/Toolchain.cmake" \ - -B "$BUILD_DIR" . -cmake --build "$BUILD_DIR" -j -cmake --install "$BUILD_DIR" --prefix "$DIST_DIR" +if [ -z "${MODULE_PATH:-}" ]; then + cmake --preset=ci-release \ + -DHTTPD_DATADOG_ENABLE_RUM=ON \ + -DCMAKE_TOOLCHAIN_FILE="/sysroot/${ARCH}-none-linux-musl/Toolchain.cmake" \ + -B "$BUILD_DIR" . + cmake --build "$BUILD_DIR" -j + cmake --install "$BUILD_DIR" --prefix "$DIST_DIR" + MODULE_PATH="$DIST_DIR/lib/mod_datadog.so" +fi cd test/integration-test @@ -39,6 +47,6 @@ uv sync # its value in a single token and avoids that race. uv run pytest \ --bin-path=/httpd/httpd-build/bin/apachectl \ - --module-path="$DIST_DIR/lib/mod_datadog.so" \ + --module-path="$MODULE_PATH" \ --log-dir="$REPO_ROOT/logs" \ -v "$@" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b7f8077..a6f0d70 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,6 @@ variables: CI_DOCKER_IMAGE_BASE: registry.ddbuild.io/ci/httpd-datadog + NYDUS_DD_IMAGE: registry.ddbuild.io/images/nydus:v98111448-6d7c7d0-v2.3.8-dd GIT_SUBMODULE_STRATEGY: recursive GIT_SUBMODULE_DEPTH: 1 GIT_CONFIG_COUNT: 1 @@ -20,7 +21,7 @@ stages: # Template jobs (hidden, reusable) .build-template: stage: build - image: $CI_DOCKER_IMAGE_BASE:$CI_IMAGE_HASH + image: $DEVCONTAINER_IMAGE script: - git config --global --add safe.directory $PWD - > @@ -42,14 +43,20 @@ stages: .test-rum-template: stage: test - image: $CI_DOCKER_IMAGE_BASE:$CI_IMAGE_HASH + image: $DEVCONTAINER_IMAGE script: - cd $CI_PROJECT_DIR/test/integration-test && uv sync && uv run pytest scenarios --module-path $CI_PROJECT_DIR/artifacts/$ARCH/mod_datadog.so --bin-path /httpd/httpd-build/bin/apachectl --log-dir $CI_PROJECT_DIR/logs -m requires_rum -v -build-ci-image: +# Build (or reuse) the devcontainer image per arch: hash the inputs, skip if +# the registry already has the tag, otherwise buildx build + push + nydusify +# in-place. The nydusified per-arch images are then stitched into a +# multi-arch manifest by `create-ci-manifest` below. +devcontainer-image: stage: ci-image tags: ["arch:$ARCH"] - image: registry.ddbuild.io/images/docker:27.3.1 + image: $NYDUS_DD_IMAGE + variables: + DDSIGN_SKIP_SIGNING: "true" parallel: matrix: - ARCH: amd64 @@ -60,14 +67,15 @@ build-ci-image: - | FILES=$(find .devcontainer/Dockerfile deps/nginx-datadog/build_env/ scripts/setup-httpd.py -type f | sort) HASH=$(for f in $FILES; do echo "--- $f ---"; cat "$f"; done | sha256sum | cut -d ' ' -f 1) - - echo "CI_IMAGE_HASH=$HASH" >> ci-image.env - IMAGE_TAG="$CI_DOCKER_IMAGE_BASE/$ARCH:$HASH" + - echo "CI_IMAGE_HASH=$HASH" >> ci-image.env + - echo "DEVCONTAINER_IMAGE=$CI_DOCKER_IMAGE_BASE:$HASH" >> ci-image.env - | - if docker buildx imagetools inspect "$IMAGE_TAG" > /dev/null 2>&1; then + if crane manifest "$IMAGE_TAG" >/dev/null 2>&1; then echo "Image $IMAGE_TAG already exists. Skipping build." else docker buildx build \ - --tag $IMAGE_TAG \ + --tag "$IMAGE_TAG" \ --platform linux/$ARCH \ --build-arg ARCH=$TOOLCHAIN_ARCH \ --push \ @@ -75,20 +83,36 @@ build-ci-image: . echo "Image $IMAGE_TAG built for $ARCH." fi + - | + MANIFEST=$(crane manifest "$IMAGE_TAG" 2>/dev/null) || { + echo "WARNING: failed to fetch manifest for $IMAGE_TAG — skipping nydus conversion" + exit 0 + } + if echo "$MANIFEST" | jq -e '[.layers[].mediaType] | any(contains("nydus"))' >/dev/null 2>&1; then + echo "$IMAGE_TAG already has nydus layers. Skipping conversion." + else + echo "Converting $IMAGE_TAG to nydus in place..." + nydus-convert "$IMAGE_TAG" "$IMAGE_TAG" || \ + echo "WARNING: nydus conversion failed — continuing without nydus layers" + fi artifacts: reports: dotenv: ci-image.env +# Combine the per-arch (nydusified) images into a single multi-arch manifest +# at $CI_DOCKER_IMAGE_BASE:$CI_IMAGE_HASH so downstream jobs can pin by tag +# alone and let the runtime pick the right arch. create-ci-manifest: stage: ci-image needs: - - build-ci-image + - devcontainer-image tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:27.3.1 + image: $NYDUS_DD_IMAGE script: - - MANIFEST_TAG="$CI_DOCKER_IMAGE_BASE:$CI_IMAGE_HASH" + - MANIFEST_TAG="$DEVCONTAINER_IMAGE" - | - if docker buildx imagetools inspect "$MANIFEST_TAG" > /dev/null 2>&1; then + if crane manifest "$MANIFEST_TAG" >/dev/null 2>&1 && \ + docker buildx imagetools inspect "$MANIFEST_TAG" --format '{{json .Manifest}}' 2>/dev/null | jq -e '.manifests | length >= 2' >/dev/null 2>&1; then echo "Multi-arch manifest $MANIFEST_TAG already exists. Skipping." else docker buildx imagetools create \ @@ -97,41 +121,10 @@ create-ci-manifest: "$CI_DOCKER_IMAGE_BASE/arm64:$CI_IMAGE_HASH" fi -nydusify-ci-image: - stage: ci-image - needs: - - build-ci-image - - create-ci-manifest - allow_failure: true - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/nydus:v98111448-6d7c7d0-v2.3.8-dd - variables: - DDSIGN_SKIP_SIGNING: "true" - script: - - | - IMAGE="$CI_DOCKER_IMAGE_BASE:$CI_IMAGE_HASH" - REPO="${IMAGE%:*}" - - # Resolve a platform manifest digest from the multi-arch index - DIGEST=$(crane manifest "$IMAGE" 2>/dev/null | jq -r '(.manifests // [])[0].digest // empty' 2>/dev/null) - if [ -n "$DIGEST" ]; then - MANIFEST=$(crane manifest "${REPO}@${DIGEST}" 2>/dev/null) - else - MANIFEST=$(crane manifest "$IMAGE" 2>/dev/null) - fi - - # Check if the platform manifest already contains nydus layers - if echo "$MANIFEST" | jq -e '[.layers[].mediaType] | any(contains("nydus"))' > /dev/null 2>&1; then - echo "$IMAGE is already nydusified. Skipping." - else - echo "Converting $IMAGE with nydusify..." - nydus-convert "$IMAGE" "$IMAGE" - fi - build:amd64: extends: .build-template needs: - - build-ci-image + - devcontainer-image - create-ci-manifest variables: ARCH: amd64 @@ -141,7 +134,7 @@ build:amd64: build:arm64: extends: .build-template needs: - - build-ci-image + - devcontainer-image - create-ci-manifest variables: ARCH: arm64 @@ -151,7 +144,7 @@ build:arm64: test-rum:amd64: extends: .test-rum-template needs: - - build-ci-image + - devcontainer-image - job: "build:amd64" variables: ARCH: amd64 @@ -160,7 +153,7 @@ test-rum:amd64: test-rum:arm64: extends: .test-rum-template needs: - - build-ci-image + - devcontainer-image - job: "build:arm64" variables: ARCH: arm64 From d30fcd7472b6be717c8b32ad6ddbb1a157dab2ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chojnacki?= Date: Mon, 20 Apr 2026 19:55:00 +0200 Subject: [PATCH 03/16] Extract devcontainer CI jobs to .gitlab/devcontainer.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the `devcontainer-image` build-and-nydusify job plus the `create-ci-manifest` multi-arch assembly into their own include file `.gitlab/devcontainer.yml`, and reference it from the main `.gitlab-ci.yml` via `include: - local:`. The NYDUS_DD_IMAGE variable is declared inside the include since it's only used by these jobs. Groups the devcontainer image pipeline into a single-responsibility file separate from the build/test/upload orchestration. Matches the pattern established by inject-browser-sdk's `.gitlab/devcontainer.yml` and keeps the main `.gitlab-ci.yml` readable (drops 72 lines). No behavioral change — job names, dependencies, env exports, and runtime semantics are identical; only file layout moves. --- .gitlab-ci.yml | 78 ++----------------------------------- .gitlab/devcontainer.yml | 83 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 75 deletions(-) create mode 100644 .gitlab/devcontainer.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a6f0d70..466a07b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,8 @@ +include: + - local: .gitlab/devcontainer.yml + variables: CI_DOCKER_IMAGE_BASE: registry.ddbuild.io/ci/httpd-datadog - NYDUS_DD_IMAGE: registry.ddbuild.io/images/nydus:v98111448-6d7c7d0-v2.3.8-dd GIT_SUBMODULE_STRATEGY: recursive GIT_SUBMODULE_DEPTH: 1 GIT_CONFIG_COUNT: 1 @@ -47,80 +49,6 @@ stages: script: - cd $CI_PROJECT_DIR/test/integration-test && uv sync && uv run pytest scenarios --module-path $CI_PROJECT_DIR/artifacts/$ARCH/mod_datadog.so --bin-path /httpd/httpd-build/bin/apachectl --log-dir $CI_PROJECT_DIR/logs -m requires_rum -v -# Build (or reuse) the devcontainer image per arch: hash the inputs, skip if -# the registry already has the tag, otherwise buildx build + push + nydusify -# in-place. The nydusified per-arch images are then stitched into a -# multi-arch manifest by `create-ci-manifest` below. -devcontainer-image: - stage: ci-image - tags: ["arch:$ARCH"] - image: $NYDUS_DD_IMAGE - variables: - DDSIGN_SKIP_SIGNING: "true" - parallel: - matrix: - - ARCH: amd64 - TOOLCHAIN_ARCH: x86_64 - - ARCH: arm64 - TOOLCHAIN_ARCH: aarch64 - script: - - | - FILES=$(find .devcontainer/Dockerfile deps/nginx-datadog/build_env/ scripts/setup-httpd.py -type f | sort) - HASH=$(for f in $FILES; do echo "--- $f ---"; cat "$f"; done | sha256sum | cut -d ' ' -f 1) - - IMAGE_TAG="$CI_DOCKER_IMAGE_BASE/$ARCH:$HASH" - - echo "CI_IMAGE_HASH=$HASH" >> ci-image.env - - echo "DEVCONTAINER_IMAGE=$CI_DOCKER_IMAGE_BASE:$HASH" >> ci-image.env - - | - if crane manifest "$IMAGE_TAG" >/dev/null 2>&1; then - echo "Image $IMAGE_TAG already exists. Skipping build." - else - docker buildx build \ - --tag "$IMAGE_TAG" \ - --platform linux/$ARCH \ - --build-arg ARCH=$TOOLCHAIN_ARCH \ - --push \ - --file .devcontainer/Dockerfile \ - . - echo "Image $IMAGE_TAG built for $ARCH." - fi - - | - MANIFEST=$(crane manifest "$IMAGE_TAG" 2>/dev/null) || { - echo "WARNING: failed to fetch manifest for $IMAGE_TAG — skipping nydus conversion" - exit 0 - } - if echo "$MANIFEST" | jq -e '[.layers[].mediaType] | any(contains("nydus"))' >/dev/null 2>&1; then - echo "$IMAGE_TAG already has nydus layers. Skipping conversion." - else - echo "Converting $IMAGE_TAG to nydus in place..." - nydus-convert "$IMAGE_TAG" "$IMAGE_TAG" || \ - echo "WARNING: nydus conversion failed — continuing without nydus layers" - fi - artifacts: - reports: - dotenv: ci-image.env - -# Combine the per-arch (nydusified) images into a single multi-arch manifest -# at $CI_DOCKER_IMAGE_BASE:$CI_IMAGE_HASH so downstream jobs can pin by tag -# alone and let the runtime pick the right arch. -create-ci-manifest: - stage: ci-image - needs: - - devcontainer-image - tags: ["arch:amd64"] - image: $NYDUS_DD_IMAGE - script: - - MANIFEST_TAG="$DEVCONTAINER_IMAGE" - - | - if crane manifest "$MANIFEST_TAG" >/dev/null 2>&1 && \ - docker buildx imagetools inspect "$MANIFEST_TAG" --format '{{json .Manifest}}' 2>/dev/null | jq -e '.manifests | length >= 2' >/dev/null 2>&1; then - echo "Multi-arch manifest $MANIFEST_TAG already exists. Skipping." - else - docker buildx imagetools create \ - --tag "$MANIFEST_TAG" \ - "$CI_DOCKER_IMAGE_BASE/amd64:$CI_IMAGE_HASH" \ - "$CI_DOCKER_IMAGE_BASE/arm64:$CI_IMAGE_HASH" - fi - build:amd64: extends: .build-template needs: diff --git a/.gitlab/devcontainer.yml b/.gitlab/devcontainer.yml new file mode 100644 index 0000000..7410b09 --- /dev/null +++ b/.gitlab/devcontainer.yml @@ -0,0 +1,83 @@ +--- +# Build pipeline for the devcontainer image used by every downstream CI +# job that compiles mod_datadog or runs integration tests. The image is +# content-addressed: its tag is a sha256 over the Dockerfile plus the +# upstream toolchain files it pulls from, so the jobs here only rebuild +# when one of those inputs actually changes. + +variables: + NYDUS_DD_IMAGE: registry.ddbuild.io/images/nydus:v98111448-6d7c7d0-v2.3.8-dd + +# Build (or reuse) the devcontainer image per arch: hash the inputs, skip if +# the registry already has the tag, otherwise buildx build + push + nydusify +# in-place. The nydusified per-arch images are then stitched into a +# multi-arch manifest by `create-ci-manifest` below. +devcontainer-image: + stage: ci-image + tags: ["arch:$ARCH"] + image: $NYDUS_DD_IMAGE + variables: + DDSIGN_SKIP_SIGNING: "true" + parallel: + matrix: + - ARCH: amd64 + TOOLCHAIN_ARCH: x86_64 + - ARCH: arm64 + TOOLCHAIN_ARCH: aarch64 + script: + - | + FILES=$(find .devcontainer/Dockerfile deps/nginx-datadog/build_env/ scripts/setup-httpd.py -type f | sort) + HASH=$(for f in $FILES; do echo "--- $f ---"; cat "$f"; done | sha256sum | cut -d ' ' -f 1) + - IMAGE_TAG="$CI_DOCKER_IMAGE_BASE/$ARCH:$HASH" + - echo "CI_IMAGE_HASH=$HASH" >> ci-image.env + - echo "DEVCONTAINER_IMAGE=$CI_DOCKER_IMAGE_BASE:$HASH" >> ci-image.env + - | + if crane manifest "$IMAGE_TAG" >/dev/null 2>&1; then + echo "Image $IMAGE_TAG already exists. Skipping build." + else + docker buildx build \ + --tag "$IMAGE_TAG" \ + --platform linux/$ARCH \ + --build-arg ARCH=$TOOLCHAIN_ARCH \ + --push \ + --file .devcontainer/Dockerfile \ + . + echo "Image $IMAGE_TAG built for $ARCH." + fi + - | + MANIFEST=$(crane manifest "$IMAGE_TAG" 2>/dev/null) || { + echo "WARNING: failed to fetch manifest for $IMAGE_TAG — skipping nydus conversion" + exit 0 + } + if echo "$MANIFEST" | jq -e '[.layers[].mediaType] | any(contains("nydus"))' >/dev/null 2>&1; then + echo "$IMAGE_TAG already has nydus layers. Skipping conversion." + else + echo "Converting $IMAGE_TAG to nydus in place..." + nydus-convert "$IMAGE_TAG" "$IMAGE_TAG" || \ + echo "WARNING: nydus conversion failed — continuing without nydus layers" + fi + artifacts: + reports: + dotenv: ci-image.env + +# Combine the per-arch (nydusified) images into a single multi-arch manifest +# at $CI_DOCKER_IMAGE_BASE:$CI_IMAGE_HASH so downstream jobs can pin by tag +# alone and let the runtime pick the right arch. +create-ci-manifest: + stage: ci-image + needs: + - devcontainer-image + tags: ["arch:amd64"] + image: $NYDUS_DD_IMAGE + script: + - MANIFEST_TAG="$DEVCONTAINER_IMAGE" + - | + if crane manifest "$MANIFEST_TAG" >/dev/null 2>&1 && \ + docker buildx imagetools inspect "$MANIFEST_TAG" --format '{{json .Manifest}}' 2>/dev/null | jq -e '.manifests | length >= 2' >/dev/null 2>&1; then + echo "Multi-arch manifest $MANIFEST_TAG already exists. Skipping." + else + docker buildx imagetools create \ + --tag "$MANIFEST_TAG" \ + "$CI_DOCKER_IMAGE_BASE/amd64:$CI_IMAGE_HASH" \ + "$CI_DOCKER_IMAGE_BASE/arm64:$CI_IMAGE_HASH" + fi From b3853c0f3ba88b8c8d745aa98755c781a1aed688 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chojnacki?= Date: Mon, 20 Apr 2026 19:57:00 +0200 Subject: [PATCH 04/16] Scope consumer image to arch-specific devcontainer build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per-arch build/test jobs used to wait for both devcontainer-image matrix slots plus create-ci-manifest before they could start. That serialized the pipeline needlessly: the amd64 test doesn't need the arm64 image to exist, and neither needs the multi-arch manifest. Export DEVCONTAINER_IMAGE per arch (\$base/\$arch:\$hash) rather than multi-arch (\$base:\$hash), and scope each consumer's `needs:` to its own arch slot via `parallel: matrix:`. Drop create-ci-manifest from the needs list entirely — it's still built for external consumers but doesn't gate the internal pipeline. Effect: when a devcontainer input (Dockerfile, nginx-datadog build_env, setup-httpd.py) changes, the matching arch's image rebuilds, and that arch's build + test jobs pick it up as soon as they're ready; the other arch proceeds in parallel on its own rebuild. --- .gitlab-ci.yml | 30 ++++++++++++++++++++++++------ .gitlab/devcontainer.yml | 14 ++++++++++---- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 466a07b..498bcff 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -52,8 +52,12 @@ stages: build:amd64: extends: .build-template needs: - - devcontainer-image - - create-ci-manifest + - job: devcontainer-image + parallel: + matrix: + - ARCH: amd64 + TOOLCHAIN_ARCH: x86_64 + artifacts: true variables: ARCH: amd64 TOOLCHAIN_ARCH: x86_64 @@ -62,8 +66,12 @@ build:amd64: build:arm64: extends: .build-template needs: - - devcontainer-image - - create-ci-manifest + - job: devcontainer-image + parallel: + matrix: + - ARCH: arm64 + TOOLCHAIN_ARCH: aarch64 + artifacts: true variables: ARCH: arm64 TOOLCHAIN_ARCH: aarch64 @@ -72,7 +80,12 @@ build:arm64: test-rum:amd64: extends: .test-rum-template needs: - - devcontainer-image + - job: devcontainer-image + parallel: + matrix: + - ARCH: amd64 + TOOLCHAIN_ARCH: x86_64 + artifacts: true - job: "build:amd64" variables: ARCH: amd64 @@ -81,7 +94,12 @@ test-rum:amd64: test-rum:arm64: extends: .test-rum-template needs: - - devcontainer-image + - job: devcontainer-image + parallel: + matrix: + - ARCH: arm64 + TOOLCHAIN_ARCH: aarch64 + artifacts: true - job: "build:arm64" variables: ARCH: arm64 diff --git a/.gitlab/devcontainer.yml b/.gitlab/devcontainer.yml index 7410b09..8f0f464 100644 --- a/.gitlab/devcontainer.yml +++ b/.gitlab/devcontainer.yml @@ -30,7 +30,11 @@ devcontainer-image: HASH=$(for f in $FILES; do echo "--- $f ---"; cat "$f"; done | sha256sum | cut -d ' ' -f 1) - IMAGE_TAG="$CI_DOCKER_IMAGE_BASE/$ARCH:$HASH" - echo "CI_IMAGE_HASH=$HASH" >> ci-image.env - - echo "DEVCONTAINER_IMAGE=$CI_DOCKER_IMAGE_BASE:$HASH" >> ci-image.env + # Export DEVCONTAINER_IMAGE as the arch-specific ref so each consumer + # runs on its own arch's image without blocking on create-ci-manifest. + # Downstream jobs scope their `needs:` to their matrix slot to pick up + # the matching value. + - echo "DEVCONTAINER_IMAGE=$IMAGE_TAG" >> ci-image.env - | if crane manifest "$IMAGE_TAG" >/dev/null 2>&1; then echo "Image $IMAGE_TAG already exists. Skipping build." @@ -61,8 +65,10 @@ devcontainer-image: dotenv: ci-image.env # Combine the per-arch (nydusified) images into a single multi-arch manifest -# at $CI_DOCKER_IMAGE_BASE:$CI_IMAGE_HASH so downstream jobs can pin by tag -# alone and let the runtime pick the right arch. +# at $CI_DOCKER_IMAGE_BASE:$CI_IMAGE_HASH so external consumers (other +# pipelines, GitHub Actions) can pin by the bare tag. Internal build/test +# jobs reference the per-arch image directly via $DEVCONTAINER_IMAGE and +# don't wait for this job. create-ci-manifest: stage: ci-image needs: @@ -70,7 +76,7 @@ create-ci-manifest: tags: ["arch:amd64"] image: $NYDUS_DD_IMAGE script: - - MANIFEST_TAG="$DEVCONTAINER_IMAGE" + - MANIFEST_TAG="$CI_DOCKER_IMAGE_BASE:$CI_IMAGE_HASH" - | if crane manifest "$MANIFEST_TAG" >/dev/null 2>&1 && \ docker buildx imagetools inspect "$MANIFEST_TAG" --format '{{json .Manifest}}' 2>/dev/null | jq -e '.manifests | length >= 2' >/dev/null 2>&1; then From 1b728235b64de6244fed17e27af8263ec4738803 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chojnacki?= Date: Tue, 21 Apr 2026 15:39:20 +0200 Subject: [PATCH 05/16] Polish devcontainer workflow: submodule guard, image docs, pytest flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Collects a few follow-ups to the devcontainer branch that were spotted while testing the flow end-to-end. devcontainer.mk: - Fail fast with actionable guidance when deps/nginx-datadog isn't initialized. Without this, the DEV_CONTAINER_HASH `find` emits warnings and silently hashes an incomplete input set, producing a tag that doesn't match what CI computed — the result was a confusing pull-then-fall-back-to-build loop. - Mount $(DEV_CONTAINER_GIT_MOUNT) into dev-shell so submodule init works from inside the container. - Move test-integration to the always-available section. $(DEVCONTAINER_RUN) is empty inside the container (script runs directly) and `docker run …` outside, so one definition works in both modes. .github/workflows/{dev,release,system-tests}.yml: - Replace the unhelpful "See in Makefile where this image comes from." comment with one that names the upstream (GitLab-built image from .devcontainer/Dockerfile, hash-computed by .gitlab/devcontainer.yml) and documents the bump procedure. .gitlab-ci.yml: - Switch pytest `--flag VALUE` to `--flag=VALUE` in .test-rum-template for the same reason as run-integration-tests.sh: pytest's early arg parser runs before conftest registers these custom options, so the space-separated form is read as a positional test path and skips conftest entirely. --- .devcontainer/devcontainer.mk | 26 ++++++++++++++++++++------ .github/workflows/dev.yml | 10 ++++++++-- .github/workflows/release.yml | 5 ++++- .github/workflows/system-tests.yml | 5 ++++- .gitlab-ci.yml | 6 +++++- 5 files changed, 41 insertions(+), 11 deletions(-) diff --git a/.devcontainer/devcontainer.mk b/.devcontainer/devcontainer.mk index a167eb1..fd9b7c1 100644 --- a/.devcontainer/devcontainer.mk +++ b/.devcontainer/devcontainer.mk @@ -13,11 +13,21 @@ dev-image: else +# Fail fast with actionable guidance if deps/nginx-datadog isn't populated — +# without this check, the DEV_CONTAINER_HASH `find` below emits warnings and +# silently hashes an incomplete input set, producing a tag that doesn't +# match what CI computed and triggering confusing pull-then-fall-back-to- +# build behavior. +ifeq ($(wildcard $(DEV_CONTAINER_REPO_ROOT)/deps/nginx-datadog/build_env/Toolchain.cmake.x86_64),) +$(error deps/nginx-datadog submodule is not initialized. Run: git submodule update --init --recursive) +endif + DEV_CONTAINER_REGISTRY ?= registry.ddbuild.io DEV_CONTAINER_IMAGE_NAME ?= $(DEV_CONTAINER_REGISTRY)/ci/httpd-datadog -# Match the hash computation in .gitlab-ci.yml (build-ci-image job): sha256 over -# .devcontainer/Dockerfile, deps/nginx-datadog/build_env/, and scripts/setup-httpd.py. +# Match the hash computation in .gitlab/devcontainer.yml (devcontainer-image +# job): sha256 over .devcontainer/Dockerfile, deps/nginx-datadog/build_env/, +# and scripts/setup-httpd.py. DEV_CONTAINER_HASH := $(shell cd $(DEV_CONTAINER_REPO_ROOT) && \ find .devcontainer/Dockerfile deps/nginx-datadog/build_env/ scripts/setup-httpd.py -type f | sort | \ while IFS= read -r f; do echo "--- $$f ---"; cat "$$f"; done | \ @@ -83,14 +93,11 @@ dev-image: dev-shell: dev-image docker run --rm -it \ -v $(DEV_CONTAINER_REPO_ROOT):$(DEV_CONTAINER_REPO_ROOT) \ + $(DEV_CONTAINER_GIT_MOUNT) \ -w $(DEV_CONTAINER_REPO_ROOT) \ $(DEV_CONTAINER_IMAGE) \ /bin/sh -.PHONY: test-integration -test-integration: dev-image - $(DEVCONTAINER_RUN) .devcontainer/run-integration-tests.sh - .PHONY: dev-image-clean dev-image-clean: @stale=$$(docker images --format '{{.Repository}}:{{.Tag}}' $(DEV_CONTAINER_IMAGE_NAME) 2>/dev/null | grep -v ':$(DEV_CONTAINER_TAG)$$' || true); \ @@ -102,3 +109,10 @@ dev-image-clean: fi endif + +# Always-available targets: $(DEVCONTAINER_RUN) is empty inside the container +# (the script runs directly) and `docker run …` outside, so one definition +# works in both modes. +.PHONY: test-integration +test-integration: dev-image + $(DEVCONTAINER_RUN) .devcontainer/run-integration-tests.sh diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 8fcd549..7c4bf54 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -19,7 +19,10 @@ jobs: needs: format runs-on: ubuntu-22.04 container: - # See in Makefile where this image comes from. + # Public mirror of the GitLab-built image defined in .devcontainer/Dockerfile + # (hash-computed by .gitlab/devcontainer.yml). Bump by copying a newer tag + # via `docker pull registry.ddbuild.io/ci/httpd-datadog: && docker tag + # ... && docker push datadog/docker-library:httpd-datadog-ci-`. image: datadog/docker-library:httpd-datadog-ci-28219c0ef3e00f1e3d5afcab61a73a5e9bd2a9b957d7545556711cce2a6262cd steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 @@ -44,7 +47,10 @@ jobs: needs: build runs-on: ubuntu-22.04 container: - # See in Makefile where this image comes from. + # Public mirror of the GitLab-built image defined in .devcontainer/Dockerfile + # (hash-computed by .gitlab/devcontainer.yml). Bump by copying a newer tag + # via `docker pull registry.ddbuild.io/ci/httpd-datadog: && docker tag + # ... && docker push datadog/docker-library:httpd-datadog-ci-`. image: datadog/docker-library:httpd-datadog-ci-28219c0ef3e00f1e3d5afcab61a73a5e9bd2a9b957d7545556711cce2a6262cd env: DD_ENV: ci diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7a7539b..db5a214 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,7 +5,10 @@ jobs: build: runs-on: ubuntu-22.04 container: - # See in Makefile where this image comes from. + # Public mirror of the GitLab-built image defined in .devcontainer/Dockerfile + # (hash-computed by .gitlab/devcontainer.yml). Bump by copying a newer tag + # via `docker pull registry.ddbuild.io/ci/httpd-datadog: && docker tag + # ... && docker push datadog/docker-library:httpd-datadog-ci-`. image: datadog/docker-library:httpd-datadog-ci-28219c0ef3e00f1e3d5afcab61a73a5e9bd2a9b957d7545556711cce2a6262cd steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 72414ee..3b87ccb 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -19,7 +19,10 @@ jobs: build-artifacts: runs-on: ubuntu-22.04 container: - # See in Makefile where this image comes from. + # Public mirror of the GitLab-built image defined in .devcontainer/Dockerfile + # (hash-computed by .gitlab/devcontainer.yml). Bump by copying a newer tag + # via `docker pull registry.ddbuild.io/ci/httpd-datadog: && docker tag + # ... && docker push datadog/docker-library:httpd-datadog-ci-`. image: datadog/docker-library:httpd-datadog-ci-28219c0ef3e00f1e3d5afcab61a73a5e9bd2a9b957d7545556711cce2a6262cd steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 498bcff..84d9753 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -46,8 +46,12 @@ stages: .test-rum-template: stage: test image: $DEVCONTAINER_IMAGE + # Same `--flag=VALUE` reasoning as run-integration-tests.sh: pytest's early + # arg parser runs before conftest registers these custom options, so the + # space-separated form gets read as a positional test path and skips + # conftest. script: - - cd $CI_PROJECT_DIR/test/integration-test && uv sync && uv run pytest scenarios --module-path $CI_PROJECT_DIR/artifacts/$ARCH/mod_datadog.so --bin-path /httpd/httpd-build/bin/apachectl --log-dir $CI_PROJECT_DIR/logs -m requires_rum -v + - cd $CI_PROJECT_DIR/test/integration-test && uv sync && uv run pytest scenarios --module-path=$CI_PROJECT_DIR/artifacts/$ARCH/mod_datadog.so --bin-path=/httpd/httpd-build/bin/apachectl --log-dir=$CI_PROJECT_DIR/logs -m requires_rum -v build:amd64: extends: .build-template From 5590f4f2d7ebda850da8debddccc4e96d55c4e83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chojnacki?= Date: Sat, 9 May 2026 00:23:51 +0200 Subject: [PATCH 06/16] Sync devcontainer with dd-repo-tools template Adopt the shared devcontainer template from DataDog/dd-repo-tools (repository-tools/devcontainer/) as the source of truth: drop in devcontainer.mk and .gitlab/devcontainer.yml verbatim and add the per-repo wiring (context.files allowlist, staged devcontainer.json, Makefile precondition + test-integration, .gitlab-ci.yml inputs). PER_ARCH stays on so the LLVM+musl+httpd build still runs on native amd64/arm64 runners. Image moves from registry.ddbuild.io/ci/httpd-datadog/: to registry.ddbuild.io/ci/httpd-datadog/devcontainer:-<12char>; the create-ci-manifest job is gone, downstream jobs now pin to $DEVCONTAINER_IMAGE_AMD64 / $DEVCONTAINER_IMAGE_ARM64 directly. --- .devcontainer/context.files | 21 +++ .devcontainer/devcontainer.json | 5 +- .devcontainer/devcontainer.mk | 310 +++++++++++++++++++++----------- .gitignore | 3 + .gitlab-ci.yml | 45 ++--- .gitlab/devcontainer.yml | 188 +++++++++++-------- Makefile | 16 ++ 7 files changed, 376 insertions(+), 212 deletions(-) create mode 100644 .devcontainer/context.files diff --git a/.devcontainer/context.files b/.devcontainer/context.files new file mode 100644 index 0000000..2f1fcdc --- /dev/null +++ b/.devcontainer/context.files @@ -0,0 +1,21 @@ +# Build-context allowlist for the devcontainer image. One rsync-relative +# path per line; '#' comments and blank lines are stripped before rsync +# sees them. Anything the Dockerfile COPYs has to live under one of these. +# +# Only files whose contents affect the image go in this list — anything +# that's only invoked at runtime via volume mount (e.g. +# run-integration-tests.sh) is intentionally excluded so editing it +# doesn't invalidate the cached image tag. + +# image recipe +.devcontainer/Dockerfile + +# musl/LLVM toolchain inputs (CHECKSUMS, locale.h.diff, alltypes.h.diff, +# Toolchain.cmake., glibc_compat.c) — sourced from the +# nginx-datadog submodule's build_env so the cross-compile sysroot here +# matches what nginx-datadog uses. +deps/nginx-datadog/build_env/ + +# Downloads + builds httpd 2.4 inside the image so test jobs have an +# Apache to load mod_datadog into without rebuilding it per pipeline. +scripts/setup-httpd.py diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 5c34a2a..8b9cf16 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,8 +1,9 @@ { "name": "httpd-datadog", + "initializeCommand": "make -f .devcontainer/devcontainer.mk .devcontainer-stage-context", "build": { - "dockerfile": "Dockerfile", - "context": "..", + "dockerfile": ".staged/.devcontainer/Dockerfile", + "context": ".staged", "args": { "ARCH": "${localEnv:ARCH:x86_64}" } diff --git a/.devcontainer/devcontainer.mk b/.devcontainer/devcontainer.mk index fd9b7c1..5d7af2d 100644 --- a/.devcontainer/devcontainer.mk +++ b/.devcontainer/devcontainer.mk @@ -1,118 +1,218 @@ -DEV_CONTAINER_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) -DEV_CONTAINER_REPO_ROOT := $(abspath $(DEV_CONTAINER_DIR)/..) - -# When running inside the container, tools are available directly. -# Check for /.dockerenv (Docker) or KUBERNETES_SERVICE_HOST (Kubernetes CI). -ifneq ($(or $(wildcard /.dockerenv),$(KUBERNETES_SERVICE_HOST)),) - -DEVCONTAINER_RUN := - -.PHONY: dev-image -dev-image: - @true - +# DO NOT EDIT. Synced from DataDog/dd-repo-tools by the +# dd-repo-tools devcontainer-bundle campaigner — edits here will be +# overwritten on the next run. +# Source of truth: https://github.com/DataDog/dd-repo-tools/blob/main/repository-tools/devcontainer/devcontainer.mk +# +# Inputs (set before `include`): +# DEV_CONTAINER_REPO_ROOT Absolute path to the repo root. +# DEV_CONTAINER_IMAGE_NAME Override image name. Defaults to +# registry.ddbuild.io/ci//devcontainer. +# DEV_CONTAINER_REQUIRED_PATHS Whitespace-separated paths that must exist +# (e.g. submodule checkouts). Missing -> +# $(error). +# DOCKER_BUILDX_FLAGS Extra flags appended to the local build. +# +# This file is the single source of truth for staging + hashing. The +# GitLab template (devcontainer.yml) calls into `make` so both paths +# share the same code. + +DEV_CONTAINER_REPO_ROOT ?= $(CURDIR) +_DEV_CONTAINER_REPO_NAME := $(notdir $(patsubst %/,%,$(DEV_CONTAINER_REPO_ROOT))) +DEV_CONTAINER_IMAGE_NAME ?= registry.ddbuild.io/ci/$(_DEV_CONTAINER_REPO_NAME)/devcontainer + +_DEV_CONTAINER_CONTEXT_FILES := $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/context.files +_DEV_CONTAINER_CONTEXT_FILTER := $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/context.filter +_DEV_CONTAINER_DOCKERFILE := $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/Dockerfile +_DEV_CONTAINER_STAGED := $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/.staged + +# uname -m -> docker --platform / conventional ARCH build-arg. +_DEV_CONTAINER_UNAME_M := $(shell uname -m) +ifneq ($(filter $(_DEV_CONTAINER_UNAME_M),arm64 aarch64),) + _DEV_CONTAINER_PLATFORM := linux/arm64 + _DEV_CONTAINER_ARCH := aarch64 else - -# Fail fast with actionable guidance if deps/nginx-datadog isn't populated — -# without this check, the DEV_CONTAINER_HASH `find` below emits warnings and -# silently hashes an incomplete input set, producing a tag that doesn't -# match what CI computed and triggering confusing pull-then-fall-back-to- -# build behavior. -ifeq ($(wildcard $(DEV_CONTAINER_REPO_ROOT)/deps/nginx-datadog/build_env/Toolchain.cmake.x86_64),) -$(error deps/nginx-datadog submodule is not initialized. Run: git submodule update --init --recursive) + _DEV_CONTAINER_PLATFORM := linux/amd64 + _DEV_CONTAINER_ARCH := x86_64 endif -DEV_CONTAINER_REGISTRY ?= registry.ddbuild.io -DEV_CONTAINER_IMAGE_NAME ?= $(DEV_CONTAINER_REGISTRY)/ci/httpd-datadog - -# Match the hash computation in .gitlab/devcontainer.yml (devcontainer-image -# job): sha256 over .devcontainer/Dockerfile, deps/nginx-datadog/build_env/, -# and scripts/setup-httpd.py. -DEV_CONTAINER_HASH := $(shell cd $(DEV_CONTAINER_REPO_ROOT) && \ - find .devcontainer/Dockerfile deps/nginx-datadog/build_env/ scripts/setup-httpd.py -type f | sort | \ - while IFS= read -r f; do echo "--- $$f ---"; cat "$$f"; done | \ - (command -v sha256sum >/dev/null 2>&1 && sha256sum || shasum -a 256) | cut -d' ' -f1) - -DEV_CONTAINER_TAG := $(DEV_CONTAINER_HASH) -DEV_CONTAINER_IMAGE := $(DEV_CONTAINER_IMAGE_NAME):$(DEV_CONTAINER_TAG) - -# Map host arch (uname -m) to the Dockerfile's ARCH build-arg. -DEV_CONTAINER_HOST_ARCH := $(shell uname -m) -ifeq ($(DEV_CONTAINER_HOST_ARCH),arm64) -DEV_CONTAINER_ARCH := aarch64 -DEV_CONTAINER_PLATFORM := linux/arm64 -else ifeq ($(DEV_CONTAINER_HOST_ARCH),aarch64) -DEV_CONTAINER_ARCH := aarch64 -DEV_CONTAINER_PLATFORM := linux/arm64 -else -DEV_CONTAINER_ARCH := x86_64 -DEV_CONTAINER_PLATFORM := linux/amd64 +# Pass --build-arg ARCH only if the Dockerfile actually declares it. +_DEV_CONTAINER_ARCH_ARG := $(shell grep -q '^ARG ARCH' $(_DEV_CONTAINER_DOCKERFILE) 2>/dev/null && echo --build-arg ARCH=$(_DEV_CONTAINER_ARCH)) + +# Skip everything inside the container (no nested docker, no pull). +_DEV_CONTAINER_INSIDE := $(shell [ -f /.dockerenv ] || [ -n "$$KUBERNETES_SERVICE_HOST" ] && echo 1) + +# Stage .devcontainer/context.files (+ optional context.filter) into $$ctx. +# Caller sets $$ctx (and arranges cleanup if needed). Verifies Dockerfile +# was staged. +define _dev_container_stage_into_ctx +if [ ! -f "$(_DEV_CONTAINER_CONTEXT_FILES)" ]; then \ + echo "ERROR: $(_DEV_CONTAINER_CONTEXT_FILES) not found" >&2; exit 1; \ +fi; \ +if [ -f "$(_DEV_CONTAINER_CONTEXT_FILTER)" ]; then \ + grep -Ev '^[[:space:]]*(#|$$)' "$(_DEV_CONTAINER_CONTEXT_FILES)" | \ + rsync -aR --files-from=- --filter="merge $(_DEV_CONTAINER_CONTEXT_FILTER)" "$(DEV_CONTAINER_REPO_ROOT)/" "$$ctx/"; \ +else \ + grep -Ev '^[[:space:]]*(#|$$)' "$(_DEV_CONTAINER_CONTEXT_FILES)" | \ + rsync -aR --files-from=- "$(DEV_CONTAINER_REPO_ROOT)/" "$$ctx/"; \ +fi; \ +if [ ! -f "$$ctx/.devcontainer/Dockerfile" ]; then \ + echo "ERROR: .devcontainer/Dockerfile not staged; check .devcontainer/context.files" >&2; \ + exit 1; \ +fi +endef + +# Stage into a fresh mktemp dir, cleaned up on shell exit. Sets $$ctx. +define _dev_container_stage_context +ctx=$$(mktemp -d); \ +trap 'rm -rf "$$ctx"' EXIT; \ +$(_dev_container_stage_into_ctx) +endef + +# Compute the 12-char tag hash from the staged tree at $$ctx. +define _dev_container_compute_tag +tag=$$(cd "$$ctx" && find . -type f | LC_ALL=C sort | \ + while IFS= read -r f; do printf -- '--- %s ---\n' "$$f"; cat "$$f"; done | \ + sha256sum | cut -c1-12) +endef + +# Git-worktree support: when .git is a gitdir pointer, make the common dir +# visible inside the container so `git` calls work. +_DEV_CONTAINER_GIT_COMMON_DIR := $(shell cd $(DEV_CONTAINER_REPO_ROOT) && git rev-parse --path-format=absolute --git-common-dir 2>/dev/null) +_DEV_CONTAINER_DEFAULT_GIT_DIR := $(DEV_CONTAINER_REPO_ROOT)/.git +ifneq ($(_DEV_CONTAINER_GIT_COMMON_DIR),) + ifneq ($(_DEV_CONTAINER_GIT_COMMON_DIR),$(_DEV_CONTAINER_DEFAULT_GIT_DIR)) + _DEV_CONTAINER_WORKTREE_MOUNT := -v $(_DEV_CONTAINER_GIT_COMMON_DIR):$(_DEV_CONTAINER_GIT_COMMON_DIR) + endif endif -# For git worktrees, .git is a file pointing to a gitdir outside the repo root; -# bind-mount the git common-dir so `git` works inside the container. -DEV_CONTAINER_GIT_COMMON_DIR := $(shell cd $(DEV_CONTAINER_REPO_ROOT) && git rev-parse --path-format=absolute --git-common-dir 2>/dev/null) -ifneq ($(DEV_CONTAINER_GIT_COMMON_DIR),) -ifneq ($(DEV_CONTAINER_GIT_COMMON_DIR),$(DEV_CONTAINER_REPO_ROOT)/.git) -DEV_CONTAINER_GIT_MOUNT := -v $(DEV_CONTAINER_GIT_COMMON_DIR):$(DEV_CONTAINER_GIT_COMMON_DIR) +# Precondition check for caller-declared required paths. +ifneq ($(strip $(DEV_CONTAINER_REQUIRED_PATHS)),) + _DEV_CONTAINER_MISSING_PATHS := $(foreach p,$(DEV_CONTAINER_REQUIRED_PATHS),$(if $(wildcard $(DEV_CONTAINER_REPO_ROOT)/$(p)),,$(p))) + ifneq ($(strip $(_DEV_CONTAINER_MISSING_PATHS)),) + $(error Missing required paths (did you init submodules?): $(_DEV_CONTAINER_MISSING_PATHS)) + endif endif + +.PHONY: dev-image dev-shell dev-image-tag \ + .devcontainer-stage-context .devcontainer-image-hash + +# Stage the build context into a stable path under .devcontainer/.staged/. +# Invoked from VS Code's initializeCommand (so build.context = `.staged` +# gets the same narrow tree CI and `make dev-image` use) and from +# devcontainer.yml's CI job. Hash-computing local targets keep using a +# mktemp dir for parallel safety; this one writes to a stable path +# because VS Code (and CI's downstream buildx step) need one. +.devcontainer-stage-context: + @rm -rf "$(_DEV_CONTAINER_STAGED)" + @mkdir -p "$(_DEV_CONTAINER_STAGED)" + @ctx="$(_DEV_CONTAINER_STAGED)"; $(_dev_container_stage_into_ctx) + +# Print the 12-char hash of the already-staged .devcontainer/.staged/ tree. +# Caller must have run `make .devcontainer-stage-context` first. Used by +# devcontainer.yml so the tag is computed by the same code that the local +# Makefile uses. +.devcontainer-image-hash: + @ctx="$(_DEV_CONTAINER_STAGED)"; \ + test -d "$$ctx" || { echo "ERROR: $$ctx not staged; run 'make .devcontainer-stage-context' first" >&2; exit 1; }; \ + $(_dev_container_compute_tag); \ + echo "$$tag" + +# Print the full image:tag that would be used. Useful for debugging. +# Works inside or outside a container (no docker required). +dev-image-tag: + @$(_dev_container_stage_context); \ + $(_dev_container_compute_tag); \ + echo "$(DEV_CONTAINER_IMAGE_NAME):$$tag" + +# Shared mounts for any in-container invocation: the repo root, plus the +# git common dir when running inside a worktree (so `git` works). +_DEV_CONTAINER_MOUNTS := \ + -v $(DEV_CONTAINER_REPO_ROOT):$(DEV_CONTAINER_REPO_ROOT) \ + $(_DEV_CONTAINER_WORKTREE_MOUNT) \ + -w $(DEV_CONTAINER_REPO_ROOT) + +# Public: prefix for running a scripted command in the devcontainer image. +# Includes the resolved image ref and `-i` (so stdin pipes work) but no +# `-t` (so it works under `make` / CI without a TTY). Usage: +# foo: dev-image +# $(IN_DEVCONTAINER) cmd args +# When `make` is invoked from inside a container, IN_DEVCONTAINER is empty, +# so the command runs in-place — same recipe works inside or outside. +IN_DEVCONTAINER ?= docker run --rm -i $(_DEV_CONTAINER_MOUNTS) \ + $$(cat $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/.image-ref) + +# linux/amd64 variant: same shape as IN_DEVCONTAINER but pinned to amd64, +# for running cross-compiled x86_64 binaries (e.g. Windows test exes under +# wine64) on any host. On Apple Silicon, Docker emulates via Rosetta. On +# native amd64 the platform pin is a no-op and we reuse the regular image. +# Targets that use this should depend on `dev-image-x86_64-cached`. +_X86_64_IMAGE_REF_FILE := $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/.image-ref-x86_64 +IN_DEVCONTAINER_X86_64 ?= docker run --rm -i --platform=linux/amd64 $(_DEV_CONTAINER_MOUNTS) \ + $$(cat $(_X86_64_IMAGE_REF_FILE)) + +ifeq ($(_DEV_CONTAINER_INSIDE),1) + IN_DEVCONTAINER := + IN_DEVCONTAINER_X86_64 := endif -DEVCONTAINER_RUN = docker run --rm \ - -v $(DEV_CONTAINER_REPO_ROOT):$(DEV_CONTAINER_REPO_ROOT) \ - $(DEV_CONTAINER_GIT_MOUNT) \ - -w $(DEV_CONTAINER_REPO_ROOT) \ - $(DEV_CONTAINER_IMAGE) +# Targets whose behaviour differs inside vs outside a container. +# Inside: dev-image is a no-op (already running on it); dev-shell fails +# loudly because there's no docker daemon. +ifeq ($(_DEV_CONTAINER_INSIDE),1) -# Ensure the dev container image is available locally. Prefer pulling from the -# registry (CI builds every tagged hash); fall back to a local build. -# Stale images with different tags are removed first. -.PHONY: dev-image +dev-image dev-image-x86_64: + @: + +dev-shell: + @echo "make dev-shell: already inside the devcontainer; run /bin/sh directly" >&2; exit 1 + +else # _DEV_CONTAINER_INSIDE + +# Pull the image if available; otherwise build locally from the staged context. dev-image: - @if ! docker image inspect $(DEV_CONTAINER_IMAGE) >/dev/null 2>&1; then \ - stale=$$(docker images --format '{{.Repository}}:{{.Tag}}' $(DEV_CONTAINER_IMAGE_NAME) 2>/dev/null | grep -v ':$(DEV_CONTAINER_TAG)$$' || true); \ - if [ -n "$$stale" ]; then \ - echo "Removing stale dev container image(s): $$stale"; \ - docker rmi $$stale || true; \ - fi; \ - if docker pull $(DEV_CONTAINER_IMAGE) >/dev/null 2>&1; then \ - echo "Pulled $(DEV_CONTAINER_IMAGE)"; \ - else \ - echo "Building dev container image $(DEV_CONTAINER_IMAGE)..."; \ - docker buildx build \ - --platform $(DEV_CONTAINER_PLATFORM) \ - --build-arg ARCH=$(DEV_CONTAINER_ARCH) \ - --load \ - -t $(DEV_CONTAINER_IMAGE) \ - -f $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/Dockerfile \ - $(DEV_CONTAINER_REPO_ROOT); \ - fi; \ - fi - -.PHONY: dev-shell -dev-shell: dev-image - docker run --rm -it \ - -v $(DEV_CONTAINER_REPO_ROOT):$(DEV_CONTAINER_REPO_ROOT) \ - $(DEV_CONTAINER_GIT_MOUNT) \ - -w $(DEV_CONTAINER_REPO_ROOT) \ - $(DEV_CONTAINER_IMAGE) \ - /bin/sh - -.PHONY: dev-image-clean -dev-image-clean: - @stale=$$(docker images --format '{{.Repository}}:{{.Tag}}' $(DEV_CONTAINER_IMAGE_NAME) 2>/dev/null | grep -v ':$(DEV_CONTAINER_TAG)$$' || true); \ - if [ -n "$$stale" ]; then \ - echo "Removing stale dev container image(s): $$stale"; \ - docker rmi $$stale; \ + @$(_dev_container_stage_context); \ + $(_dev_container_compute_tag); \ + ref="$(DEV_CONTAINER_IMAGE_NAME):$$tag"; \ + if docker pull "$$ref" >/dev/null 2>&1; then \ + echo "Pulled $$ref"; \ else \ - echo "No stale dev container images found."; \ - fi - + echo "Pull miss; building $$ref locally"; \ + docker buildx build \ + --platform $(_DEV_CONTAINER_PLATFORM) \ + --file "$$ctx/.devcontainer/Dockerfile" \ + --build-context repo="$$ctx" \ + $(_DEV_CONTAINER_ARCH_ARG) \ + $(DOCKER_BUILDX_FLAGS) \ + --load -t "$$ref" \ + "$$ctx"; \ + fi; \ + echo "$$ref" > $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/.image-ref + +# Build the linux/amd64 variant. On amd64 hosts this reuses dev-image; on +# arm64 hosts it produces a separate `:-x86_64` image so it doesn't +# clobber the native one. Buildx caches per-platform so repeats are cheap. +ifeq ($(_DEV_CONTAINER_PLATFORM),linux/amd64) +dev-image-x86_64: dev-image + @cp $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/.image-ref $(_X86_64_IMAGE_REF_FILE) +else +dev-image-x86_64: + @$(_dev_container_stage_context); \ + $(_dev_container_compute_tag); \ + ref="$(DEV_CONTAINER_IMAGE_NAME):$$tag-x86_64"; \ + docker buildx build --platform=linux/amd64 \ + --file "$$ctx/.devcontainer/Dockerfile" \ + --build-context repo="$$ctx" \ + --build-arg ARCH=x86_64 \ + $(DOCKER_BUILDX_FLAGS) \ + --load -t "$$ref" \ + "$$ctx"; \ + echo "$$ref" > $(_X86_64_IMAGE_REF_FILE) endif -# Always-available targets: $(DEVCONTAINER_RUN) is empty inside the container -# (the script runs directly) and `docker run …` outside, so one definition -# works in both modes. -.PHONY: test-integration -test-integration: dev-image - $(DEVCONTAINER_RUN) .devcontainer/run-integration-tests.sh +# Interactive shell in the built image. Mirrors IN_DEVCONTAINER but with +# `-it` for terminal handling. +dev-shell: dev-image + @ref=$$(cat $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/.image-ref); \ + docker run --rm -it $(_DEV_CONTAINER_MOUNTS) "$$ref" /bin/sh + +endif # _DEV_CONTAINER_INSIDE diff --git a/.gitignore b/.gitignore index 92a9c5e..cb252be 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,6 @@ test/integration-test/htmlcov/ test/integration-test/log-*/ test/integration-test/logs/ test/integration-test/uv.lock +.devcontainer/.staged/ +.devcontainer/.image-ref +.devcontainer/.image-ref-x86_64 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 84d9753..78f9123 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,12 @@ include: - local: .gitlab/devcontainer.yml variables: - CI_DOCKER_IMAGE_BASE: registry.ddbuild.io/ci/httpd-datadog + # Inputs to .gitlab/devcontainer.yml (the dd-repo-tools shared + # devcontainer template). DEVCONTAINER_PER_ARCH=true splits the image + # build into native amd64+arm64 jobs because the LLVM+musl sysroot, + # cbindgen, and httpd compile stages are too slow under qemu emulation. + DEVCONTAINER_REPO_NAME: httpd-datadog + DEVCONTAINER_PER_ARCH: "true" GIT_SUBMODULE_STRATEGY: recursive GIT_SUBMODULE_DEPTH: 1 GIT_CONFIG_COUNT: 1 @@ -15,7 +20,6 @@ default: aud: image-integrity stages: - - ci-image - build - test - aws @@ -23,7 +27,9 @@ stages: # Template jobs (hidden, reusable) .build-template: stage: build - image: $DEVCONTAINER_IMAGE + needs: + - job: devcontainer_image + artifacts: true script: - git config --global --add safe.directory $PWD - > @@ -45,7 +51,6 @@ stages: .test-rum-template: stage: test - image: $DEVCONTAINER_IMAGE # Same `--flag=VALUE` reasoning as run-integration-tests.sh: pytest's early # arg parser runs before conftest registers these custom options, so the # space-separated form gets read as a positional test path and skips @@ -55,13 +60,7 @@ stages: build:amd64: extends: .build-template - needs: - - job: devcontainer-image - parallel: - matrix: - - ARCH: amd64 - TOOLCHAIN_ARCH: x86_64 - artifacts: true + image: $DEVCONTAINER_IMAGE_AMD64 variables: ARCH: amd64 TOOLCHAIN_ARCH: x86_64 @@ -69,13 +68,7 @@ build:amd64: build:arm64: extends: .build-template - needs: - - job: devcontainer-image - parallel: - matrix: - - ARCH: arm64 - TOOLCHAIN_ARCH: aarch64 - artifacts: true + image: $DEVCONTAINER_IMAGE_ARM64 variables: ARCH: arm64 TOOLCHAIN_ARCH: aarch64 @@ -83,28 +76,24 @@ build:arm64: test-rum:amd64: extends: .test-rum-template + image: $DEVCONTAINER_IMAGE_AMD64 needs: - - job: devcontainer-image - parallel: - matrix: - - ARCH: amd64 - TOOLCHAIN_ARCH: x86_64 + - job: devcontainer_image artifacts: true - job: "build:amd64" + artifacts: true variables: ARCH: amd64 tags: ["arch:amd64"] test-rum:arm64: extends: .test-rum-template + image: $DEVCONTAINER_IMAGE_ARM64 needs: - - job: devcontainer-image - parallel: - matrix: - - ARCH: arm64 - TOOLCHAIN_ARCH: aarch64 + - job: devcontainer_image artifacts: true - job: "build:arm64" + artifacts: true variables: ARCH: arm64 tags: ["arch:arm64"] diff --git a/.gitlab/devcontainer.yml b/.gitlab/devcontainer.yml index 8f0f464..abc7ca3 100644 --- a/.gitlab/devcontainer.yml +++ b/.gitlab/devcontainer.yml @@ -1,89 +1,123 @@ ---- -# Build pipeline for the devcontainer image used by every downstream CI -# job that compiles mod_datadog or runs integration tests. The image is -# content-addressed: its tag is a sha256 over the Dockerfile plus the -# upstream toolchain files it pulls from, so the jobs here only rebuild -# when one of those inputs actually changes. +# DO NOT EDIT. Synced from DataDog/dd-repo-tools by the +# dd-repo-tools devcontainer-bundle campaigner into +# `.gitlab/devcontainer.yml` — edits here will be overwritten. +# +# Shared devcontainer pipeline. One job does everything: stage the +# context, compute the tag, check the cache, build + nydusify on miss, +# emit ci-image.env for downstream jobs. +# +# Consumer repo wires this in via: +# variables: +# DEVCONTAINER_REPO_NAME: my-repo +# # optional: +# # DEVCONTAINER_IMAGE_NAME: registry.ddbuild.io/ci/my-repo # flat path +# # DEVCONTAINER_PER_ARCH: "true" # arch matrix +# include: +# - local: .gitlab/devcontainer.yml +# +# DEVCONTAINER_PER_ARCH=true splits the build into a parallel:matrix over +# amd64 and arm64, each pinned to its native runner via tags:[arch:$ARCH]. +# Use it when the Dockerfile's `RUN` steps are slow under qemu emulation — +# httpd-datadog is the canonical case (LLVM+musl sysroot + cbindgen + +# httpd compile). Default off because most repos build fast enough that +# one multi-arch buildx run is cheaper than two native runner slots. +# +# Per-repo inputs live in: +# .devcontainer/Dockerfile the dockerfile +# .devcontainer/context.files rsync --files-from list (one path per line; +# '#'/blank lines are comments, stripped +# before being handed to rsync) +# .devcontainer/context.filter optional rsync filter rules variables: - NYDUS_DD_IMAGE: registry.ddbuild.io/images/nydus:v98111448-6d7c7d0-v2.3.8-dd + DEVCONTAINER_IMAGE_NAME: registry.ddbuild.io/ci/${DEVCONTAINER_REPO_NAME}/devcontainer + DEVCONTAINER_PER_ARCH: "" + # Bootstrap image used by the devcontainer_image job itself. Update + # DEVCONTAINER_BOOTSTRAP_IMAGE_TAG by PR after each manual + # publish-devcontainer-bootstrap-image run (same workflow as + # CI_TESTING_IMAGE_TAG). + DEVCONTAINER_BOOTSTRAP_IMAGE_BASE: registry.ddbuild.io/ci/dd-repo-tools/devcontainer-bootstrap + DEVCONTAINER_BOOTSTRAP_IMAGE_TAG: "" + DEVCONTAINER_BOOTSTRAP_IMAGE: registry.ddbuild.io/images/nydus:v2.4.0-dd.3 -# Build (or reuse) the devcontainer image per arch: hash the inputs, skip if -# the registry already has the tag, otherwise buildx build + push + nydusify -# in-place. The nydusified per-arch images are then stitched into a -# multi-arch manifest by `create-ci-manifest` below. -devcontainer-image: - stage: ci-image - tags: ["arch:$ARCH"] - image: $NYDUS_DD_IMAGE - variables: - DDSIGN_SKIP_SIGNING: "true" - parallel: - matrix: - - ARCH: amd64 - TOOLCHAIN_ARCH: x86_64 - - ARCH: arm64 - TOOLCHAIN_ARCH: aarch64 +devcontainer_image: + stage: .pre + image: $DEVCONTAINER_BOOTSTRAP_IMAGE + needs: [] + tags: ["arch:amd64"] + # Compose the bootstrap image from BASE:TAG when a tag is pinned (the + # post-`publish-devcontainer-bootstrap-image` workflow), otherwise fall + # back to the upstream nydus image defined in `variables:` above. + rules: + - if: '$DEVCONTAINER_BOOTSTRAP_IMAGE_TAG' + variables: + DEVCONTAINER_BOOTSTRAP_IMAGE: $DEVCONTAINER_BOOTSTRAP_IMAGE_BASE:$DEVCONTAINER_BOOTSTRAP_IMAGE_TAG + - when: always script: - | - FILES=$(find .devcontainer/Dockerfile deps/nginx-datadog/build_env/ scripts/setup-httpd.py -type f | sort) - HASH=$(for f in $FILES; do echo "--- $f ---"; cat "$f"; done | sha256sum | cut -d ' ' -f 1) - - IMAGE_TAG="$CI_DOCKER_IMAGE_BASE/$ARCH:$HASH" - - echo "CI_IMAGE_HASH=$HASH" >> ci-image.env - # Export DEVCONTAINER_IMAGE as the arch-specific ref so each consumer - # runs on its own arch's image without blocking on create-ci-manifest. - # Downstream jobs scope their `needs:` to their matrix slot to pick up - # the matching value. - - echo "DEVCONTAINER_IMAGE=$IMAGE_TAG" >> ci-image.env - - | - if crane manifest "$IMAGE_TAG" >/dev/null 2>&1; then - echo "Image $IMAGE_TAG already exists. Skipping build." - else - docker buildx build \ - --tag "$IMAGE_TAG" \ - --platform linux/$ARCH \ - --build-arg ARCH=$TOOLCHAIN_ARCH \ - --push \ - --file .devcontainer/Dockerfile \ - . - echo "Image $IMAGE_TAG built for $ARCH." + set -euo pipefail + + # Bootstrap image is minimal — install make (and rsync, which the + # staging step needs) on demand. Each `command -v` guards a single + # apt-get so a re-run on an image that already has them is a no-op. + missing="" + command -v make >/dev/null || missing="$missing make" + command -v rsync >/dev/null || missing="$missing rsync" + if [ -n "$missing" ]; then + apt-get update -qq + # shellcheck disable=SC2086 + apt-get install -y -qq --no-install-recommends $missing fi - - | - MANIFEST=$(crane manifest "$IMAGE_TAG" 2>/dev/null) || { - echo "WARNING: failed to fetch manifest for $IMAGE_TAG — skipping nydus conversion" - exit 0 + + # Stage + hash via devcontainer.mk so CI and local `make` use the + # same code. `-f` is explicit so we don't depend on the consumer's + # root Makefile including the fragment. .devcontainer-stage-context + # writes to .devcontainer/.staged/; .devcontainer-image-hash reads + # from the same path and prints the 12-char tag. + MK="-f .devcontainer/devcontainer.mk" + # shellcheck disable=SC2086 + make $MK .devcontainer-stage-context + ctx=.devcontainer/.staged + # shellcheck disable=SC2086 + TAG=$(make $MK -s .devcontainer-image-hash) + echo "Devcontainer tag: $TAG" + + # ---- build-or-skip per arch ---- + build_and_nydusify() { + local ref=$1 platform=$2 arch_arg=$3 + echo "Building $ref ..." + # shellcheck disable=SC2086 + docker buildx build \ + --platform "$platform" \ + --file "$ctx/.devcontainer/Dockerfile" \ + --build-context "repo=$ctx" \ + $arch_arg \ + --push -t "$ref" \ + "$ctx" + nydus-convert "$ref" "$ref" } - if echo "$MANIFEST" | jq -e '[.layers[].mediaType] | any(contains("nydus"))' >/dev/null 2>&1; then - echo "$IMAGE_TAG already has nydus layers. Skipping conversion." + + if [ -n "$DEVCONTAINER_PER_ARCH" ]; then + AMD_REF="${DEVCONTAINER_IMAGE_NAME}:amd64-${TAG}" + ARM_REF="${DEVCONTAINER_IMAGE_NAME}:arm64-${TAG}" + crane manifest "$AMD_REF" >/dev/null 2>&1 \ + || build_and_nydusify "$AMD_REF" linux/amd64 "--build-arg ARCH=x86_64" + crane manifest "$ARM_REF" >/dev/null 2>&1 \ + || build_and_nydusify "$ARM_REF" linux/arm64 "--build-arg ARCH=aarch64" + { + echo "DEVCONTAINER_IMAGE_AMD64=$AMD_REF" + echo "DEVCONTAINER_IMAGE_ARM64=$ARM_REF" + echo "CI_IMAGE_HASH=$TAG" + } > ci-image.env else - echo "Converting $IMAGE_TAG to nydus in place..." - nydus-convert "$IMAGE_TAG" "$IMAGE_TAG" || \ - echo "WARNING: nydus conversion failed — continuing without nydus layers" + REF="${DEVCONTAINER_IMAGE_NAME}:${TAG}" + crane manifest "$REF" >/dev/null 2>&1 \ + || build_and_nydusify "$REF" linux/amd64,linux/arm64 "" + { + echo "DEVCONTAINER_IMAGE=$REF" + echo "CI_IMAGE_HASH=$TAG" + } > ci-image.env fi artifacts: reports: dotenv: ci-image.env - -# Combine the per-arch (nydusified) images into a single multi-arch manifest -# at $CI_DOCKER_IMAGE_BASE:$CI_IMAGE_HASH so external consumers (other -# pipelines, GitHub Actions) can pin by the bare tag. Internal build/test -# jobs reference the per-arch image directly via $DEVCONTAINER_IMAGE and -# don't wait for this job. -create-ci-manifest: - stage: ci-image - needs: - - devcontainer-image - tags: ["arch:amd64"] - image: $NYDUS_DD_IMAGE - script: - - MANIFEST_TAG="$CI_DOCKER_IMAGE_BASE:$CI_IMAGE_HASH" - - | - if crane manifest "$MANIFEST_TAG" >/dev/null 2>&1 && \ - docker buildx imagetools inspect "$MANIFEST_TAG" --format '{{json .Manifest}}' 2>/dev/null | jq -e '.manifests | length >= 2' >/dev/null 2>&1; then - echo "Multi-arch manifest $MANIFEST_TAG already exists. Skipping." - else - docker buildx imagetools create \ - --tag "$MANIFEST_TAG" \ - "$CI_DOCKER_IMAGE_BASE/amd64:$CI_IMAGE_HASH" \ - "$CI_DOCKER_IMAGE_BASE/arm64:$CI_IMAGE_HASH" - fi diff --git a/Makefile b/Makefile index 3b5374c..2c1b44e 100644 --- a/Makefile +++ b/Makefile @@ -1 +1,17 @@ +# Pin the image name so it doesn't drift with the worktree directory name +# (the upstream default is derived from $(notdir $(CURDIR)) — fine for a +# canonical clone called `httpd-datadog`, wrong for any feature-branch +# worktree). Matches DEVCONTAINER_IMAGE_NAME in .gitlab-ci.yml. +DEV_CONTAINER_IMAGE_NAME := registry.ddbuild.io/ci/httpd-datadog/devcontainer + +# The devcontainer Dockerfile pulls toolchain files from the +# nginx-datadog submodule's build_env/. Without this guard the staging +# step silently produces a tree missing those files and the resulting +# tag mismatches what CI computes — fail loudly instead. +DEV_CONTAINER_REQUIRED_PATHS := deps/nginx-datadog/build_env/Toolchain.cmake.x86_64 + include .devcontainer/devcontainer.mk + +.PHONY: test-integration +test-integration: dev-image + $(IN_DEVCONTAINER) .devcontainer/run-integration-tests.sh From 5190892511a4a6f0e30af6829c9d6c228d6bd408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chojnacki?= Date: Sat, 9 May 2026 00:29:41 +0200 Subject: [PATCH 07/16] Address remaining review comments on devcontainer PR - run-integration-tests.sh: wipe DIST_DIR before `cmake --install` so removed/renamed build outputs don't linger and silently get used by tests (xlamorlette). - Restore image-provenance discoverability from the Makefile and de-duplicate the bump procedure that was repeated in four workflow files. The procedure now lives in .github/workflows/CI_IMAGE.md; each workflow keeps a single-line pointer next to its `image:` ref. A new `make mirror-public-image` target stages the context, computes the same hash GitLab uses, pulls the amd64 variant from registry.ddbuild.io, retags it as datadog/docker-library, and prints the exact `image:` value to copy into the workflows (Copilot, xlamorlette). --- .devcontainer/run-integration-tests.sh | 3 ++ .github/workflows/CI_IMAGE.md | 44 ++++++++++++++++++++++++++ .github/workflows/dev.yml | 10 ++---- .github/workflows/release.yml | 5 +-- .github/workflows/system-tests.yml | 5 +-- Makefile | 21 ++++++++++++ 6 files changed, 72 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/CI_IMAGE.md diff --git a/.devcontainer/run-integration-tests.sh b/.devcontainer/run-integration-tests.sh index 84579ac..cfcd025 100755 --- a/.devcontainer/run-integration-tests.sh +++ b/.devcontainer/run-integration-tests.sh @@ -28,6 +28,9 @@ if [ -z "${MODULE_PATH:-}" ]; then -DCMAKE_TOOLCHAIN_FILE="/sysroot/${ARCH}-none-linux-musl/Toolchain.cmake" \ -B "$BUILD_DIR" . cmake --build "$BUILD_DIR" -j + # Wipe DIST_DIR so a removed/renamed file in the build doesn't + # linger from a previous install and silently get used by tests. + rm -rf "$DIST_DIR" cmake --install "$BUILD_DIR" --prefix "$DIST_DIR" MODULE_PATH="$DIST_DIR/lib/mod_datadog.so" fi diff --git a/.github/workflows/CI_IMAGE.md b/.github/workflows/CI_IMAGE.md new file mode 100644 index 0000000..4c5b056 --- /dev/null +++ b/.github/workflows/CI_IMAGE.md @@ -0,0 +1,44 @@ +# CI image used by GitHub Actions + +The workflows in this directory pin `image:` to +`datadog/docker-library:httpd-datadog-ci-` — a public Docker Hub +mirror of the internal devcontainer image built by GitLab CI from +[`.devcontainer/Dockerfile`](../../.devcontainer/Dockerfile). The build ++ caching pipeline lives in [`.gitlab/devcontainer.yml`](../../.gitlab/devcontainer.yml); +the image-tag hash is the sha256 of the staged build context defined +by [`.devcontainer/context.files`](../../.devcontainer/context.files). + +GitHub-hosted runners can't pull from `registry.ddbuild.io`, so we +periodically copy the latest amd64 variant to Docker Hub and bump the +`image:` reference in each workflow. + +## Bumping the image + +Once GitLab CI has published a new tag (any change to a path listed in +`.devcontainer/context.files` invalidates the cache and triggers a +rebuild), mirror it from the repo root: + +```sh +make mirror-public-image +``` + +The target stages the build context, computes the same 12-char hash +GitLab uses, pulls `registry.ddbuild.io/ci/httpd-datadog/devcontainer:amd64-`, +retags it as `datadog/docker-library:httpd-datadog-ci-`, and +pushes. It then prints the exact `image:` value to copy into: + +- `dev.yml` (build + test jobs) +- `release.yml` (build job) +- `system-tests.yml` (build-artifacts job) + +## Why two image namespaces + +| Consumer | Pull from | Why | +|-----------------|----------------------------------------------------------------------------|-----| +| GitLab CI | `registry.ddbuild.io/ci/httpd-datadog/devcontainer:-` | private, fast, nydus-optimized | +| GitHub Actions | `datadog/docker-library:httpd-datadog-ci-` (mirrored from the above) | public, reachable from GitHub-hosted runners | +| Local `make` | Either of the above (pull `registry.ddbuild.io/...` if you have access; build locally otherwise) | dev convenience | + +The two mirrors stay in sync only as far as the most recent +`make mirror-public-image` run — bump the workflows promptly after +running it. diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 7c4bf54..3591300 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -19,10 +19,7 @@ jobs: needs: format runs-on: ubuntu-22.04 container: - # Public mirror of the GitLab-built image defined in .devcontainer/Dockerfile - # (hash-computed by .gitlab/devcontainer.yml). Bump by copying a newer tag - # via `docker pull registry.ddbuild.io/ci/httpd-datadog: && docker tag - # ... && docker push datadog/docker-library:httpd-datadog-ci-`. + # See .github/workflows/CI_IMAGE.md — bump via `make mirror-public-image`. image: datadog/docker-library:httpd-datadog-ci-28219c0ef3e00f1e3d5afcab61a73a5e9bd2a9b957d7545556711cce2a6262cd steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 @@ -47,10 +44,7 @@ jobs: needs: build runs-on: ubuntu-22.04 container: - # Public mirror of the GitLab-built image defined in .devcontainer/Dockerfile - # (hash-computed by .gitlab/devcontainer.yml). Bump by copying a newer tag - # via `docker pull registry.ddbuild.io/ci/httpd-datadog: && docker tag - # ... && docker push datadog/docker-library:httpd-datadog-ci-`. + # See .github/workflows/CI_IMAGE.md — bump via `make mirror-public-image`. image: datadog/docker-library:httpd-datadog-ci-28219c0ef3e00f1e3d5afcab61a73a5e9bd2a9b957d7545556711cce2a6262cd env: DD_ENV: ci diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index db5a214..9be06b5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,10 +5,7 @@ jobs: build: runs-on: ubuntu-22.04 container: - # Public mirror of the GitLab-built image defined in .devcontainer/Dockerfile - # (hash-computed by .gitlab/devcontainer.yml). Bump by copying a newer tag - # via `docker pull registry.ddbuild.io/ci/httpd-datadog: && docker tag - # ... && docker push datadog/docker-library:httpd-datadog-ci-`. + # See .github/workflows/CI_IMAGE.md — bump via `make mirror-public-image`. image: datadog/docker-library:httpd-datadog-ci-28219c0ef3e00f1e3d5afcab61a73a5e9bd2a9b957d7545556711cce2a6262cd steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 3b87ccb..8202138 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -19,10 +19,7 @@ jobs: build-artifacts: runs-on: ubuntu-22.04 container: - # Public mirror of the GitLab-built image defined in .devcontainer/Dockerfile - # (hash-computed by .gitlab/devcontainer.yml). Bump by copying a newer tag - # via `docker pull registry.ddbuild.io/ci/httpd-datadog: && docker tag - # ... && docker push datadog/docker-library:httpd-datadog-ci-`. + # See .github/workflows/CI_IMAGE.md — bump via `make mirror-public-image`. image: datadog/docker-library:httpd-datadog-ci-28219c0ef3e00f1e3d5afcab61a73a5e9bd2a9b957d7545556711cce2a6262cd steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 diff --git a/Makefile b/Makefile index 2c1b44e..8e69d03 100644 --- a/Makefile +++ b/Makefile @@ -15,3 +15,24 @@ include .devcontainer/devcontainer.mk .PHONY: test-integration test-integration: dev-image $(IN_DEVCONTAINER) .devcontainer/run-integration-tests.sh + +# GitHub-hosted runners can't pull from registry.ddbuild.io; the workflows +# in .github/workflows/ pin to a public Docker Hub mirror at +# datadog/docker-library:httpd-datadog-ci-. After a new tag is built +# by .gitlab/devcontainer.yml (any change to .devcontainer/context.files +# inputs invalidates the cache), run this target to copy the amd64 variant +# to Docker Hub and bump the workflows. +# See .github/workflows/CI_IMAGE.md for the bump procedure. +.PHONY: mirror-public-image +mirror-public-image: + @$(MAKE) -s -f .devcontainer/devcontainer.mk .devcontainer-stage-context + @hash=$$($(MAKE) -s -f .devcontainer/devcontainer.mk .devcontainer-image-hash); \ + src="$(DEV_CONTAINER_IMAGE_NAME):amd64-$$hash"; \ + public="datadog/docker-library:httpd-datadog-ci-$$hash"; \ + echo "Mirroring $$src -> $$public"; \ + docker pull --platform linux/amd64 "$$src"; \ + docker tag "$$src" "$$public"; \ + docker push "$$public"; \ + echo ""; \ + echo "Update image: in .github/workflows/{dev,release,system-tests}.yml to:"; \ + echo " $$public" From 404a02a1c5aee4507a344f739d02da41c2a86129 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chojnacki?= Date: Sat, 9 May 2026 00:32:52 +0200 Subject: [PATCH 08/16] Consolidate workflow build steps into `make ci-build` The dev/release/system-tests workflows each inlined the same safe.directory + submodule init + cmake configure/build/install sequence, with the only variation being the preset (ci-dev vs ci-release). Move the recipe into a single `ci-build` make target parameterized by PRESET, and replace the four-step block in each workflow with one `make ci-build PRESET=...` step. Restores discoverability ("how do these images build?") to the Makefile that Copilot flagged was lost in the original PR, and removes the duplication xlamorlette pointed at. --- .github/workflows/dev.yml | 10 +--------- .github/workflows/release.yml | 10 +--------- .github/workflows/system-tests.yml | 10 +--------- Makefile | 20 ++++++++++++++++++++ 4 files changed, 23 insertions(+), 27 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 3591300..a551fa5 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -23,16 +23,8 @@ jobs: image: datadog/docker-library:httpd-datadog-ci-28219c0ef3e00f1e3d5afcab61a73a5e9bd2a9b957d7545556711cce2a6262cd steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - - name: Add cloned repo as safe - run: sh -c "git config --global --add safe.directory $PWD" - - name: Init required submodules - run: git submodule update --init --depth=1 deps/dd-trace-cpp deps/nginx-datadog - - name: Configure - run: cmake --preset=ci-dev -B build . - name: Build - run: | - cmake --build build -j --verbose - cmake --install build --prefix dist + run: make ci-build PRESET=ci-dev - name: Export library uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9be06b5..6514dd7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,16 +9,8 @@ jobs: image: datadog/docker-library:httpd-datadog-ci-28219c0ef3e00f1e3d5afcab61a73a5e9bd2a9b957d7545556711cce2a6262cd steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - - name: Add cloned repo as safe - run: sh -c "git config --global --add safe.directory $PWD" - - name: Init required submodules - run: git submodule update --init --depth=1 deps/dd-trace-cpp deps/nginx-datadog - - name: Configure - run: cmake --preset ci-release -B build . - name: Build - run: | - cmake --build build -j --verbose - cmake --install build --prefix dist + run: make ci-build PRESET=ci-release - name: Export library uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 8202138..dbde772 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -23,16 +23,8 @@ jobs: image: datadog/docker-library:httpd-datadog-ci-28219c0ef3e00f1e3d5afcab61a73a5e9bd2a9b957d7545556711cce2a6262cd steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - - name: Add cloned repo as safe - run: sh -c "git config --global --add safe.directory $PWD" - - name: Init required submodules - run: git submodule update --init --depth=1 deps/dd-trace-cpp deps/nginx-datadog - - name: Configure - run: cmake --preset=ci-dev -B build . - name: Build - run: | - cmake --build build -j --verbose - cmake --install build --prefix dist + run: make ci-build PRESET=ci-dev - name: Export library uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: diff --git a/Makefile b/Makefile index 8e69d03..996de54 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,26 @@ include .devcontainer/devcontainer.mk test-integration: dev-image $(IN_DEVCONTAINER) .devcontainer/run-integration-tests.sh +# One-shot CI build: configure + build + install mod_datadog. Used by +# every job in .github/workflows/ that produces the shared library +# artifact (dev/release/system-tests). Assumes it runs inside the +# pinned devcontainer image (toolchain + httpd already present); +# callers pick the preset via PRESET=ci-dev|ci-release. +# +# `safe.directory` is a noop on hosts but required when actions/checkout +# clones into a path GitHub's runner UID doesn't own — the case for our +# container jobs. The submodule list deliberately omits inject-browser-sdk; +# the GitHub workflows here build without RUM (HTTPD_DATADOG_ENABLE_RUM=OFF +# by default in the cmake presets), so pulling it would be wasted bytes. +PRESET ?= ci-dev +.PHONY: ci-build +ci-build: + git config --global --add safe.directory $(CURDIR) + git submodule update --init --depth=1 deps/dd-trace-cpp deps/nginx-datadog + cmake --preset=$(PRESET) -B build . + cmake --build build -j --verbose + cmake --install build --prefix dist + # GitHub-hosted runners can't pull from registry.ddbuild.io; the workflows # in .github/workflows/ pin to a public Docker Hub mirror at # datadog/docker-library:httpd-datadog-ci-. After a new tag is built From 50c5700612a81e6960c2f388ba58a6db5334c0dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chojnacki?= Date: Sat, 9 May 2026 00:38:28 +0200 Subject: [PATCH 09/16] Add dev-image-cached fast-path; gate submodule guard for ci-build - dev-image-cached: skip the stage+hash+docker-pull cycle when .devcontainer/.image-ref already exists. Used by test-integration so hot dev loops don't re-stage on every run; matches the pattern in inject-browser-sdk/uber_stack. `make dev-image` still always refreshes. - Skip DEV_CONTAINER_REQUIRED_PATHS for ci-build only. ci-build runs its own `git submodule update` inline (the GitHub workflows checkout without submodules), so the parse-time guard would otherwise fail before ci-build gets a chance to init them. Other targets keep the guard. --- Makefile | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 996de54..2efa022 100644 --- a/Makefile +++ b/Makefile @@ -8,25 +8,41 @@ DEV_CONTAINER_IMAGE_NAME := registry.ddbuild.io/ci/httpd-datadog/devcontainer # nginx-datadog submodule's build_env/. Without this guard the staging # step silently produces a tree missing those files and the resulting # tag mismatches what CI computes — fail loudly instead. +# +# Skipped for `ci-build`, which runs its own `git submodule update` +# inline (the GitHub workflows checkout without submodules and init +# them as part of the build). All other targets (dev-image, +# test-integration, mirror-public-image) genuinely need build_env +# already present and keep the guard. +ifneq ($(MAKECMDGOALS),ci-build) DEV_CONTAINER_REQUIRED_PATHS := deps/nginx-datadog/build_env/Toolchain.cmake.x86_64 +endif include .devcontainer/devcontainer.mk +# Fast-path for hot callers (test-integration, pre-commit hooks, …): +# skip the stage+hash+docker-pull cycle when .image-ref is already +# resolved. Run `make dev-image` explicitly to refresh after a +# Dockerfile or context.files change. Lives here (not in devcontainer.mk) +# because that file is upstream-synced. +.PHONY: dev-image-cached +ifeq ($(_DEV_CONTAINER_INSIDE),1) +dev-image-cached: + @: +else +dev-image-cached: + @[ -f $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/.image-ref ] || $(MAKE) dev-image +endif + .PHONY: test-integration -test-integration: dev-image +test-integration: dev-image-cached $(IN_DEVCONTAINER) .devcontainer/run-integration-tests.sh -# One-shot CI build: configure + build + install mod_datadog. Used by -# every job in .github/workflows/ that produces the shared library -# artifact (dev/release/system-tests). Assumes it runs inside the -# pinned devcontainer image (toolchain + httpd already present); -# callers pick the preset via PRESET=ci-dev|ci-release. -# -# `safe.directory` is a noop on hosts but required when actions/checkout -# clones into a path GitHub's runner UID doesn't own — the case for our -# container jobs. The submodule list deliberately omits inject-browser-sdk; -# the GitHub workflows here build without RUM (HTTPD_DATADOG_ENABLE_RUM=OFF -# by default in the cmake presets), so pulling it would be wasted bytes. +# One-shot CI build: trust the workdir, init the submodules cmake +# actually consumes, configure/build/install. Used by every job in +# .github/workflows/ that produces mod_datadog.so. Switch presets via +# PRESET=ci-release. inject-browser-sdk is omitted from the submodule +# list because RUM is off by default (HTTPD_DATADOG_ENABLE_RUM=OFF). PRESET ?= ci-dev .PHONY: ci-build ci-build: From b1b96977496c024d2ea0d26e07d9ce163430d49d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chojnacki?= Date: Sat, 9 May 2026 00:45:15 +0200 Subject: [PATCH 10/16] Drop dev-image-cached variant The fast-path target added complexity for an unproven win: test-integration isn't called in tight enough loops to need it, and `make dev-image` is already a no-op when the image is already pulled (the docker daemon checks manifest existence before re-pulling). Keep the test-integration target hitting `dev-image` directly. --- Makefile | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 2efa022..63610c0 100644 --- a/Makefile +++ b/Makefile @@ -20,22 +20,8 @@ endif include .devcontainer/devcontainer.mk -# Fast-path for hot callers (test-integration, pre-commit hooks, …): -# skip the stage+hash+docker-pull cycle when .image-ref is already -# resolved. Run `make dev-image` explicitly to refresh after a -# Dockerfile or context.files change. Lives here (not in devcontainer.mk) -# because that file is upstream-synced. -.PHONY: dev-image-cached -ifeq ($(_DEV_CONTAINER_INSIDE),1) -dev-image-cached: - @: -else -dev-image-cached: - @[ -f $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/.image-ref ] || $(MAKE) dev-image -endif - .PHONY: test-integration -test-integration: dev-image-cached +test-integration: dev-image $(IN_DEVCONTAINER) .devcontainer/run-integration-tests.sh # One-shot CI build: trust the workdir, init the submodules cmake From 19deb97c5e1b3abc05a9c20481c3dfb49c9cae9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chojnacki?= Date: Sat, 9 May 2026 00:52:40 +0200 Subject: [PATCH 11/16] Re-sync devcontainer template from dd-repo-tools/shared/devcontainer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pull the current main of DataDog/dd-repo-tools/shared/devcontainer. Behavioral parity verified: dev-image-tag returns the same 12-char hash as before the bump. Notable upstream changes: - devcontainer.mk: rsync staging collapsed from an if/else into a single call gated by an optional `$filter_arg`; internal var rename `_X86_64_IMAGE_REF_FILE` -> `_DEV_CONTAINER_X86_64_IMAGE_REF_FILE`; `dev-image-x86_64` added to the .PHONY list. - devcontainer.yml: PER_ARCH semantic clarified — both arch builds run sequentially in the same .pre job pinned to arch:amd64 (the arm64 build is qemu-emulated). Native parallelism for the image build itself is future work upstream. Updated the consumer comment in .gitlab-ci.yml accordingly: PER_ARCH=true still earns its keep here because every downstream build / test job pulls the arch-matched ref and runs natively, which is what actually matters for the slow cmake compile + integration suite. --- .devcontainer/devcontainer.mk | 30 +++++++++++++----------------- .gitlab-ci.yml | 8 +++++--- .gitlab/devcontainer.yml | 21 +++++++++++---------- 3 files changed, 29 insertions(+), 30 deletions(-) diff --git a/.devcontainer/devcontainer.mk b/.devcontainer/devcontainer.mk index 5d7af2d..790c6e7 100644 --- a/.devcontainer/devcontainer.mk +++ b/.devcontainer/devcontainer.mk @@ -1,7 +1,7 @@ # DO NOT EDIT. Synced from DataDog/dd-repo-tools by the # dd-repo-tools devcontainer-bundle campaigner — edits here will be # overwritten on the next run. -# Source of truth: https://github.com/DataDog/dd-repo-tools/blob/main/repository-tools/devcontainer/devcontainer.mk +# Source of truth: https://github.com/DataDog/dd-repo-tools/blob/main/shared/devcontainer/devcontainer.mk # # Inputs (set before `include`): # DEV_CONTAINER_REPO_ROOT Absolute path to the repo root. @@ -48,13 +48,10 @@ define _dev_container_stage_into_ctx if [ ! -f "$(_DEV_CONTAINER_CONTEXT_FILES)" ]; then \ echo "ERROR: $(_DEV_CONTAINER_CONTEXT_FILES) not found" >&2; exit 1; \ fi; \ -if [ -f "$(_DEV_CONTAINER_CONTEXT_FILTER)" ]; then \ - grep -Ev '^[[:space:]]*(#|$$)' "$(_DEV_CONTAINER_CONTEXT_FILES)" | \ - rsync -aR --files-from=- --filter="merge $(_DEV_CONTAINER_CONTEXT_FILTER)" "$(DEV_CONTAINER_REPO_ROOT)/" "$$ctx/"; \ -else \ - grep -Ev '^[[:space:]]*(#|$$)' "$(_DEV_CONTAINER_CONTEXT_FILES)" | \ - rsync -aR --files-from=- "$(DEV_CONTAINER_REPO_ROOT)/" "$$ctx/"; \ -fi; \ +filter_arg=; \ +[ -f "$(_DEV_CONTAINER_CONTEXT_FILTER)" ] && filter_arg="--filter=merge $(_DEV_CONTAINER_CONTEXT_FILTER)"; \ +grep -Ev '^[[:space:]]*(#|$$)' "$(_DEV_CONTAINER_CONTEXT_FILES)" | \ + rsync -aR --files-from=- $$filter_arg "$(DEV_CONTAINER_REPO_ROOT)/" "$$ctx/"; \ if [ ! -f "$$ctx/.devcontainer/Dockerfile" ]; then \ echo "ERROR: .devcontainer/Dockerfile not staged; check .devcontainer/context.files" >&2; \ exit 1; \ @@ -93,7 +90,7 @@ ifneq ($(strip $(DEV_CONTAINER_REQUIRED_PATHS)),) endif endif -.PHONY: dev-image dev-shell dev-image-tag \ +.PHONY: dev-image dev-image-x86_64 dev-shell dev-image-tag \ .devcontainer-stage-context .devcontainer-image-hash # Stage the build context into a stable path under .devcontainer/.staged/. @@ -145,10 +142,9 @@ IN_DEVCONTAINER ?= docker run --rm -i $(_DEV_CONTAINER_MOUNTS) \ # for running cross-compiled x86_64 binaries (e.g. Windows test exes under # wine64) on any host. On Apple Silicon, Docker emulates via Rosetta. On # native amd64 the platform pin is a no-op and we reuse the regular image. -# Targets that use this should depend on `dev-image-x86_64-cached`. -_X86_64_IMAGE_REF_FILE := $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/.image-ref-x86_64 +_DEV_CONTAINER_X86_64_IMAGE_REF_FILE := $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/.image-ref-x86_64 IN_DEVCONTAINER_X86_64 ?= docker run --rm -i --platform=linux/amd64 $(_DEV_CONTAINER_MOUNTS) \ - $$(cat $(_X86_64_IMAGE_REF_FILE)) + $$(cat $(_DEV_CONTAINER_X86_64_IMAGE_REF_FILE)) ifeq ($(_DEV_CONTAINER_INSIDE),1) IN_DEVCONTAINER := @@ -190,10 +186,12 @@ dev-image: # Build the linux/amd64 variant. On amd64 hosts this reuses dev-image; on # arm64 hosts it produces a separate `:-x86_64` image so it doesn't -# clobber the native one. Buildx caches per-platform so repeats are cheap. +# clobber the native one — built locally only (CI doesn't publish the +# `-x86_64` ref), so no docker pull attempt. Buildx caches per-platform +# so repeats are cheap. ifeq ($(_DEV_CONTAINER_PLATFORM),linux/amd64) dev-image-x86_64: dev-image - @cp $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/.image-ref $(_X86_64_IMAGE_REF_FILE) + @cp $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/.image-ref $(_DEV_CONTAINER_X86_64_IMAGE_REF_FILE) else dev-image-x86_64: @$(_dev_container_stage_context); \ @@ -206,11 +204,9 @@ dev-image-x86_64: $(DOCKER_BUILDX_FLAGS) \ --load -t "$$ref" \ "$$ctx"; \ - echo "$$ref" > $(_X86_64_IMAGE_REF_FILE) + echo "$$ref" > $(_DEV_CONTAINER_X86_64_IMAGE_REF_FILE) endif -# Interactive shell in the built image. Mirrors IN_DEVCONTAINER but with -# `-it` for terminal handling. dev-shell: dev-image @ref=$$(cat $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/.image-ref); \ docker run --rm -it $(_DEV_CONTAINER_MOUNTS) "$$ref" /bin/sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 78f9123..9e9b3b6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,9 +3,11 @@ include: variables: # Inputs to .gitlab/devcontainer.yml (the dd-repo-tools shared - # devcontainer template). DEVCONTAINER_PER_ARCH=true splits the image - # build into native amd64+arm64 jobs because the LLVM+musl sysroot, - # cbindgen, and httpd compile stages are too slow under qemu emulation. + # devcontainer template). DEVCONTAINER_PER_ARCH=true publishes + # $IMAGE:amd64-$TAG and $IMAGE:arm64-$TAG; the downstream build / test + # jobs below then pull the matching arch and run on native runners + # (`tags: ["arch:$ARCH"]`). The cmake compile + integration-test suite + # is too slow under qemu to run any other way. DEVCONTAINER_REPO_NAME: httpd-datadog DEVCONTAINER_PER_ARCH: "true" GIT_SUBMODULE_STRATEGY: recursive diff --git a/.gitlab/devcontainer.yml b/.gitlab/devcontainer.yml index abc7ca3..9a55880 100644 --- a/.gitlab/devcontainer.yml +++ b/.gitlab/devcontainer.yml @@ -15,12 +15,14 @@ # include: # - local: .gitlab/devcontainer.yml # -# DEVCONTAINER_PER_ARCH=true splits the build into a parallel:matrix over -# amd64 and arm64, each pinned to its native runner via tags:[arch:$ARCH]. -# Use it when the Dockerfile's `RUN` steps are slow under qemu emulation — -# httpd-datadog is the canonical case (LLVM+musl sysroot + cbindgen + -# httpd compile). Default off because most repos build fast enough that -# one multi-arch buildx run is cheaper than two native runner slots. +# DEVCONTAINER_PER_ARCH=true builds two single-arch refs ($IMAGE:amd64-$TAG +# and $IMAGE:arm64-$TAG) instead of one multi-arch manifest. Downstream +# jobs pulling the matching ref can then run on a native-arch runner, +# avoiding qemu emulation at consume time. Both builds still happen +# sequentially in this single .pre job pinned to arch:amd64 — moving the +# build itself to a parallel:matrix on native runners is future work. +# Default off because most repos build fast enough that the single +# multi-arch buildx run is the cheaper option. # # Per-repo inputs live in: # .devcontainer/Dockerfile the dockerfile @@ -32,10 +34,9 @@ variables: DEVCONTAINER_IMAGE_NAME: registry.ddbuild.io/ci/${DEVCONTAINER_REPO_NAME}/devcontainer DEVCONTAINER_PER_ARCH: "" - # Bootstrap image used by the devcontainer_image job itself. Update - # DEVCONTAINER_BOOTSTRAP_IMAGE_TAG by PR after each manual - # publish-devcontainer-bootstrap-image run (same workflow as - # CI_TESTING_IMAGE_TAG). + # Bootstrap image used by the devcontainer_image job. The signed-retag + # flow (DEVCONTAINER_BOOTSTRAP_IMAGE_TAG) is wired up but the producing + # job is on hold along with template publishing. DEVCONTAINER_BOOTSTRAP_IMAGE_BASE: registry.ddbuild.io/ci/dd-repo-tools/devcontainer-bootstrap DEVCONTAINER_BOOTSTRAP_IMAGE_TAG: "" DEVCONTAINER_BOOTSTRAP_IMAGE: registry.ddbuild.io/images/nydus:v2.4.0-dd.3 From 798b321c55b9754bfff2f04e354bbac00c62b97c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chojnacki?= Date: Sat, 9 May 2026 00:59:09 +0200 Subject: [PATCH 12/16] Pin DEVCONTAINER_BOOTSTRAP_IMAGE to previously-working nydus build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream's default bootstrap image (registry.ddbuild.io/images/nydus:v2.4.0-dd.3) fails get_sources with `terminal prompts disabled` when GitLab tries to clone submodules via gitretriever — auth handshake never happens. The hash-pinned image the pre-sync devcontainer-image job ran on doesn't have that problem, so override the template default until dd-repo-tools ships a bootstrap that handles gitretriever auth out of the box. Pipeline #377 (commit 1b72823, pre-sync) succeeded on the same submodule config; #418/#419/#420 (post-sync) all fail at the same get_sources step. --- .gitlab-ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9e9b3b6..180ecae 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,6 +10,14 @@ variables: # is too slow under qemu to run any other way. DEVCONTAINER_REPO_NAME: httpd-datadog DEVCONTAINER_PER_ARCH: "true" + # Pin the bootstrap image to the hash-tagged nydus build that the + # previous (hand-rolled) devcontainer job ran on. The upstream default + # (registry.ddbuild.io/images/nydus:v2.4.0-dd.3) doesn't have whatever + # gitretriever credential helper get_sources needs for submodule clones, + # so it dies with `terminal prompts disabled` before our script runs + # (see pipeline #420). Drop this override once dd-repo-tools' bootstrap + # image (or its retag flow) ships with the right auth. + DEVCONTAINER_BOOTSTRAP_IMAGE: registry.ddbuild.io/images/nydus:v98111448-6d7c7d0-v2.3.8-dd GIT_SUBMODULE_STRATEGY: recursive GIT_SUBMODULE_DEPTH: 1 GIT_CONFIG_COUNT: 1 From d050c3bbb7b8c50cb7229ce19884793d77ccc1a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chojnacki?= Date: Sat, 9 May 2026 01:03:36 +0200 Subject: [PATCH 13/16] Rewrite gitretriever submodule URLs to authenticated gitlab mirror MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pipeline #420 (and #418/#419) failed in get_sources with `fatal: could not read Username for 'http://gitretriever...'` on every submodule clone. Same failure pattern as pipeline #377 (the last pre-sync success), but the warning differs: #377 saw "Falling back to GitLab on next attempt" and recovered; #420 sees "Fallback to GitLab/Gitaly is disabled, retrying gitretriever". DDbuild's runner config stopped allowing the gitlab/gitaly fallback when gitretriever auth fails. Add a second `url..insteadOf` rule pointed at the gitretriever base URL so git rewrites submodule URLs to the authenticated gitlab.ddbuild.io mirror up front, never attempting the broken gitretriever path. Drops the bootstrap-image pin from the previous commit — that override was based on a wrong hypothesis (the failure is in get_sources, before the bootstrap image is even used). --- .gitlab-ci.yml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 180ecae..9e91b11 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,19 +10,21 @@ variables: # is too slow under qemu to run any other way. DEVCONTAINER_REPO_NAME: httpd-datadog DEVCONTAINER_PER_ARCH: "true" - # Pin the bootstrap image to the hash-tagged nydus build that the - # previous (hand-rolled) devcontainer job ran on. The upstream default - # (registry.ddbuild.io/images/nydus:v2.4.0-dd.3) doesn't have whatever - # gitretriever credential helper get_sources needs for submodule clones, - # so it dies with `terminal prompts disabled` before our script runs - # (see pipeline #420). Drop this override once dd-repo-tools' bootstrap - # image (or its retag flow) ships with the right auth. - DEVCONTAINER_BOOTSTRAP_IMAGE: registry.ddbuild.io/images/nydus:v98111448-6d7c7d0-v2.3.8-dd GIT_SUBMODULE_STRATEGY: recursive GIT_SUBMODULE_DEPTH: 1 - GIT_CONFIG_COUNT: 1 + # Two insteadOf rules pointed at the same authenticated gitlab.ddbuild.io + # base. The first catches GitHub URLs (legacy, used by some submodule + # configs); the second catches the gitretriever URL the runner rewrites + # relative submodule paths to — gitretriever's fetch path requires an + # auth helper that isn't available to plain `git clone`, and DDbuild + # has stopped letting get_sources fall back to GitLab/Gitaly when that + # fails (see pipeline #420). Rewriting the URL up-front lets the + # submodule clones go directly through the authenticated mirror. + GIT_CONFIG_COUNT: 2 GIT_CONFIG_KEY_0: url.https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/.insteadOf GIT_CONFIG_VALUE_0: https://github.com/DataDog/ + GIT_CONFIG_KEY_1: url.https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/.insteadOf + GIT_CONFIG_VALUE_1: http://gitretriever.codesync.all-clusters.local-dc.fabric.dog:8080/DataDog/ default: id_tokens: From 98eb05476e4c9539c1e7bef70a3b5131403d3833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chojnacki?= Date: Sat, 9 May 2026 01:04:18 +0200 Subject: [PATCH 14/16] Revert "Rewrite gitretriever submodule URLs to authenticated gitlab mirror" This reverts commit d050c3bbb7b8c50cb7229ce19884793d77ccc1a8. --- .gitlab-ci.yml | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9e91b11..180ecae 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,21 +10,19 @@ variables: # is too slow under qemu to run any other way. DEVCONTAINER_REPO_NAME: httpd-datadog DEVCONTAINER_PER_ARCH: "true" + # Pin the bootstrap image to the hash-tagged nydus build that the + # previous (hand-rolled) devcontainer job ran on. The upstream default + # (registry.ddbuild.io/images/nydus:v2.4.0-dd.3) doesn't have whatever + # gitretriever credential helper get_sources needs for submodule clones, + # so it dies with `terminal prompts disabled` before our script runs + # (see pipeline #420). Drop this override once dd-repo-tools' bootstrap + # image (or its retag flow) ships with the right auth. + DEVCONTAINER_BOOTSTRAP_IMAGE: registry.ddbuild.io/images/nydus:v98111448-6d7c7d0-v2.3.8-dd GIT_SUBMODULE_STRATEGY: recursive GIT_SUBMODULE_DEPTH: 1 - # Two insteadOf rules pointed at the same authenticated gitlab.ddbuild.io - # base. The first catches GitHub URLs (legacy, used by some submodule - # configs); the second catches the gitretriever URL the runner rewrites - # relative submodule paths to — gitretriever's fetch path requires an - # auth helper that isn't available to plain `git clone`, and DDbuild - # has stopped letting get_sources fall back to GitLab/Gitaly when that - # fails (see pipeline #420). Rewriting the URL up-front lets the - # submodule clones go directly through the authenticated mirror. - GIT_CONFIG_COUNT: 2 + GIT_CONFIG_COUNT: 1 GIT_CONFIG_KEY_0: url.https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/.insteadOf GIT_CONFIG_VALUE_0: https://github.com/DataDog/ - GIT_CONFIG_KEY_1: url.https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/.insteadOf - GIT_CONFIG_VALUE_1: http://gitretriever.codesync.all-clusters.local-dc.fabric.dog:8080/DataDog/ default: id_tokens: From 7aa1cc96ede2a8ead473d90ab6297414f600109d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chojnacki?= Date: Sat, 9 May 2026 01:04:18 +0200 Subject: [PATCH 15/16] Revert "Pin DEVCONTAINER_BOOTSTRAP_IMAGE to previously-working nydus build" This reverts commit 798b321c55b9754bfff2f04e354bbac00c62b97c. --- .gitlab-ci.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 180ecae..9e9b3b6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,14 +10,6 @@ variables: # is too slow under qemu to run any other way. DEVCONTAINER_REPO_NAME: httpd-datadog DEVCONTAINER_PER_ARCH: "true" - # Pin the bootstrap image to the hash-tagged nydus build that the - # previous (hand-rolled) devcontainer job ran on. The upstream default - # (registry.ddbuild.io/images/nydus:v2.4.0-dd.3) doesn't have whatever - # gitretriever credential helper get_sources needs for submodule clones, - # so it dies with `terminal prompts disabled` before our script runs - # (see pipeline #420). Drop this override once dd-repo-tools' bootstrap - # image (or its retag flow) ships with the right auth. - DEVCONTAINER_BOOTSTRAP_IMAGE: registry.ddbuild.io/images/nydus:v98111448-6d7c7d0-v2.3.8-dd GIT_SUBMODULE_STRATEGY: recursive GIT_SUBMODULE_DEPTH: 1 GIT_CONFIG_COUNT: 1 From e1034cfe06c4b3a1ef48ba3c722a43a7b8c262d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chojnacki?= Date: Sat, 9 May 2026 01:07:07 +0200 Subject: [PATCH 16/16] Use absolute GitHub URLs for submodules so insteadOf rewrite matches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Relative submodule URLs (`../foo.git`) get resolved against the parent repo's URL — which on DDbuild is the gitretriever URL the runner uses to fetch the main repo. The existing `url.https://...@gitlab.ddbuild.io/DataDog/.insteadOf` rule in .gitlab-ci.yml only rewrites `https://github.com/DataDog/`, so the gitretriever-resolved submodule URLs slipped past it and hit `fatal: could not read Username` in get_sources (pipeline #420). Switching the submodule URLs to absolute github.com URLs makes the existing insteadOf rule actually match, so submodule clones go through the authenticated gitlab.ddbuild.io mirror. Same fix landed in DataDog/nginx-datadog#366 (commit fc330c9). --- .gitmodules | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 52a8a84..52183b5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,10 +1,10 @@ [submodule "deps/dd-trace-cpp"] path = deps/dd-trace-cpp - url = ../dd-trace-cpp.git + url = https://github.com/DataDog/dd-trace-cpp.git [submodule "deps/nginx-datadog"] path = deps/nginx-datadog - url = ../httpd-datadog.git + url = https://github.com/DataDog/httpd-datadog.git branch = mirror-nginx-datadog [submodule "deps/inject-browser-sdk"] path = deps/inject-browser-sdk - url = ../inject-browser-sdk.git + url = https://github.com/DataDog/inject-browser-sdk.git