diff --git a/.gitlab/Dockerfile b/.devcontainer/Dockerfile similarity index 100% rename from .gitlab/Dockerfile rename to .devcontainer/Dockerfile diff --git a/.devcontainer/context.files b/.devcontainer/context.files new file mode 100644 index 0000000..2f1fcdc --- /dev/null +++ b/.devcontainer/context.files @@ -0,0 +1,21 @@ +# Build-context allowlist for the devcontainer image. One rsync-relative +# path per line; '#' comments and blank lines are stripped before rsync +# sees them. Anything the Dockerfile COPYs has to live under one of these. +# +# Only files whose contents affect the image go in this list — anything +# that's only invoked at runtime via volume mount (e.g. +# run-integration-tests.sh) is intentionally excluded so editing it +# doesn't invalidate the cached image tag. + +# image recipe +.devcontainer/Dockerfile + +# musl/LLVM toolchain inputs (CHECKSUMS, locale.h.diff, alltypes.h.diff, +# Toolchain.cmake., glibc_compat.c) — sourced from the +# nginx-datadog submodule's build_env so the cross-compile sysroot here +# matches what nginx-datadog uses. +deps/nginx-datadog/build_env/ + +# Downloads + builds httpd 2.4 inside the image so test jobs have an +# Apache to load mod_datadog into without rebuilding it per pipeline. +scripts/setup-httpd.py diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..8b9cf16 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,11 @@ +{ + "name": "httpd-datadog", + "initializeCommand": "make -f .devcontainer/devcontainer.mk .devcontainer-stage-context", + "build": { + "dockerfile": ".staged/.devcontainer/Dockerfile", + "context": ".staged", + "args": { + "ARCH": "${localEnv:ARCH:x86_64}" + } + } +} diff --git a/.devcontainer/devcontainer.mk b/.devcontainer/devcontainer.mk new file mode 100644 index 0000000..790c6e7 --- /dev/null +++ b/.devcontainer/devcontainer.mk @@ -0,0 +1,214 @@ +# DO NOT EDIT. Synced from DataDog/dd-repo-tools by the +# dd-repo-tools devcontainer-bundle campaigner — edits here will be +# overwritten on the next run. +# Source of truth: https://github.com/DataDog/dd-repo-tools/blob/main/shared/devcontainer/devcontainer.mk +# +# Inputs (set before `include`): +# DEV_CONTAINER_REPO_ROOT Absolute path to the repo root. +# DEV_CONTAINER_IMAGE_NAME Override image name. Defaults to +# registry.ddbuild.io/ci//devcontainer. +# DEV_CONTAINER_REQUIRED_PATHS Whitespace-separated paths that must exist +# (e.g. submodule checkouts). Missing -> +# $(error). +# DOCKER_BUILDX_FLAGS Extra flags appended to the local build. +# +# This file is the single source of truth for staging + hashing. The +# GitLab template (devcontainer.yml) calls into `make` so both paths +# share the same code. + +DEV_CONTAINER_REPO_ROOT ?= $(CURDIR) +_DEV_CONTAINER_REPO_NAME := $(notdir $(patsubst %/,%,$(DEV_CONTAINER_REPO_ROOT))) +DEV_CONTAINER_IMAGE_NAME ?= registry.ddbuild.io/ci/$(_DEV_CONTAINER_REPO_NAME)/devcontainer + +_DEV_CONTAINER_CONTEXT_FILES := $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/context.files +_DEV_CONTAINER_CONTEXT_FILTER := $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/context.filter +_DEV_CONTAINER_DOCKERFILE := $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/Dockerfile +_DEV_CONTAINER_STAGED := $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/.staged + +# uname -m -> docker --platform / conventional ARCH build-arg. +_DEV_CONTAINER_UNAME_M := $(shell uname -m) +ifneq ($(filter $(_DEV_CONTAINER_UNAME_M),arm64 aarch64),) + _DEV_CONTAINER_PLATFORM := linux/arm64 + _DEV_CONTAINER_ARCH := aarch64 +else + _DEV_CONTAINER_PLATFORM := linux/amd64 + _DEV_CONTAINER_ARCH := x86_64 +endif + +# Pass --build-arg ARCH only if the Dockerfile actually declares it. +_DEV_CONTAINER_ARCH_ARG := $(shell grep -q '^ARG ARCH' $(_DEV_CONTAINER_DOCKERFILE) 2>/dev/null && echo --build-arg ARCH=$(_DEV_CONTAINER_ARCH)) + +# Skip everything inside the container (no nested docker, no pull). +_DEV_CONTAINER_INSIDE := $(shell [ -f /.dockerenv ] || [ -n "$$KUBERNETES_SERVICE_HOST" ] && echo 1) + +# Stage .devcontainer/context.files (+ optional context.filter) into $$ctx. +# Caller sets $$ctx (and arranges cleanup if needed). Verifies Dockerfile +# was staged. +define _dev_container_stage_into_ctx +if [ ! -f "$(_DEV_CONTAINER_CONTEXT_FILES)" ]; then \ + echo "ERROR: $(_DEV_CONTAINER_CONTEXT_FILES) not found" >&2; exit 1; \ +fi; \ +filter_arg=; \ +[ -f "$(_DEV_CONTAINER_CONTEXT_FILTER)" ] && filter_arg="--filter=merge $(_DEV_CONTAINER_CONTEXT_FILTER)"; \ +grep -Ev '^[[:space:]]*(#|$$)' "$(_DEV_CONTAINER_CONTEXT_FILES)" | \ + rsync -aR --files-from=- $$filter_arg "$(DEV_CONTAINER_REPO_ROOT)/" "$$ctx/"; \ +if [ ! -f "$$ctx/.devcontainer/Dockerfile" ]; then \ + echo "ERROR: .devcontainer/Dockerfile not staged; check .devcontainer/context.files" >&2; \ + exit 1; \ +fi +endef + +# Stage into a fresh mktemp dir, cleaned up on shell exit. Sets $$ctx. +define _dev_container_stage_context +ctx=$$(mktemp -d); \ +trap 'rm -rf "$$ctx"' EXIT; \ +$(_dev_container_stage_into_ctx) +endef + +# Compute the 12-char tag hash from the staged tree at $$ctx. +define _dev_container_compute_tag +tag=$$(cd "$$ctx" && find . -type f | LC_ALL=C sort | \ + while IFS= read -r f; do printf -- '--- %s ---\n' "$$f"; cat "$$f"; done | \ + sha256sum | cut -c1-12) +endef + +# Git-worktree support: when .git is a gitdir pointer, make the common dir +# visible inside the container so `git` calls work. +_DEV_CONTAINER_GIT_COMMON_DIR := $(shell cd $(DEV_CONTAINER_REPO_ROOT) && git rev-parse --path-format=absolute --git-common-dir 2>/dev/null) +_DEV_CONTAINER_DEFAULT_GIT_DIR := $(DEV_CONTAINER_REPO_ROOT)/.git +ifneq ($(_DEV_CONTAINER_GIT_COMMON_DIR),) + ifneq ($(_DEV_CONTAINER_GIT_COMMON_DIR),$(_DEV_CONTAINER_DEFAULT_GIT_DIR)) + _DEV_CONTAINER_WORKTREE_MOUNT := -v $(_DEV_CONTAINER_GIT_COMMON_DIR):$(_DEV_CONTAINER_GIT_COMMON_DIR) + endif +endif + +# Precondition check for caller-declared required paths. +ifneq ($(strip $(DEV_CONTAINER_REQUIRED_PATHS)),) + _DEV_CONTAINER_MISSING_PATHS := $(foreach p,$(DEV_CONTAINER_REQUIRED_PATHS),$(if $(wildcard $(DEV_CONTAINER_REPO_ROOT)/$(p)),,$(p))) + ifneq ($(strip $(_DEV_CONTAINER_MISSING_PATHS)),) + $(error Missing required paths (did you init submodules?): $(_DEV_CONTAINER_MISSING_PATHS)) + endif +endif + +.PHONY: dev-image dev-image-x86_64 dev-shell dev-image-tag \ + .devcontainer-stage-context .devcontainer-image-hash + +# Stage the build context into a stable path under .devcontainer/.staged/. +# Invoked from VS Code's initializeCommand (so build.context = `.staged` +# gets the same narrow tree CI and `make dev-image` use) and from +# devcontainer.yml's CI job. Hash-computing local targets keep using a +# mktemp dir for parallel safety; this one writes to a stable path +# because VS Code (and CI's downstream buildx step) need one. +.devcontainer-stage-context: + @rm -rf "$(_DEV_CONTAINER_STAGED)" + @mkdir -p "$(_DEV_CONTAINER_STAGED)" + @ctx="$(_DEV_CONTAINER_STAGED)"; $(_dev_container_stage_into_ctx) + +# Print the 12-char hash of the already-staged .devcontainer/.staged/ tree. +# Caller must have run `make .devcontainer-stage-context` first. Used by +# devcontainer.yml so the tag is computed by the same code that the local +# Makefile uses. +.devcontainer-image-hash: + @ctx="$(_DEV_CONTAINER_STAGED)"; \ + test -d "$$ctx" || { echo "ERROR: $$ctx not staged; run 'make .devcontainer-stage-context' first" >&2; exit 1; }; \ + $(_dev_container_compute_tag); \ + echo "$$tag" + +# Print the full image:tag that would be used. Useful for debugging. +# Works inside or outside a container (no docker required). +dev-image-tag: + @$(_dev_container_stage_context); \ + $(_dev_container_compute_tag); \ + echo "$(DEV_CONTAINER_IMAGE_NAME):$$tag" + +# Shared mounts for any in-container invocation: the repo root, plus the +# git common dir when running inside a worktree (so `git` works). +_DEV_CONTAINER_MOUNTS := \ + -v $(DEV_CONTAINER_REPO_ROOT):$(DEV_CONTAINER_REPO_ROOT) \ + $(_DEV_CONTAINER_WORKTREE_MOUNT) \ + -w $(DEV_CONTAINER_REPO_ROOT) + +# Public: prefix for running a scripted command in the devcontainer image. +# Includes the resolved image ref and `-i` (so stdin pipes work) but no +# `-t` (so it works under `make` / CI without a TTY). Usage: +# foo: dev-image +# $(IN_DEVCONTAINER) cmd args +# When `make` is invoked from inside a container, IN_DEVCONTAINER is empty, +# so the command runs in-place — same recipe works inside or outside. +IN_DEVCONTAINER ?= docker run --rm -i $(_DEV_CONTAINER_MOUNTS) \ + $$(cat $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/.image-ref) + +# linux/amd64 variant: same shape as IN_DEVCONTAINER but pinned to amd64, +# for running cross-compiled x86_64 binaries (e.g. Windows test exes under +# wine64) on any host. On Apple Silicon, Docker emulates via Rosetta. On +# native amd64 the platform pin is a no-op and we reuse the regular image. +_DEV_CONTAINER_X86_64_IMAGE_REF_FILE := $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/.image-ref-x86_64 +IN_DEVCONTAINER_X86_64 ?= docker run --rm -i --platform=linux/amd64 $(_DEV_CONTAINER_MOUNTS) \ + $$(cat $(_DEV_CONTAINER_X86_64_IMAGE_REF_FILE)) + +ifeq ($(_DEV_CONTAINER_INSIDE),1) + IN_DEVCONTAINER := + IN_DEVCONTAINER_X86_64 := +endif + +# Targets whose behaviour differs inside vs outside a container. +# Inside: dev-image is a no-op (already running on it); dev-shell fails +# loudly because there's no docker daemon. +ifeq ($(_DEV_CONTAINER_INSIDE),1) + +dev-image dev-image-x86_64: + @: + +dev-shell: + @echo "make dev-shell: already inside the devcontainer; run /bin/sh directly" >&2; exit 1 + +else # _DEV_CONTAINER_INSIDE + +# Pull the image if available; otherwise build locally from the staged context. +dev-image: + @$(_dev_container_stage_context); \ + $(_dev_container_compute_tag); \ + ref="$(DEV_CONTAINER_IMAGE_NAME):$$tag"; \ + if docker pull "$$ref" >/dev/null 2>&1; then \ + echo "Pulled $$ref"; \ + else \ + echo "Pull miss; building $$ref locally"; \ + docker buildx build \ + --platform $(_DEV_CONTAINER_PLATFORM) \ + --file "$$ctx/.devcontainer/Dockerfile" \ + --build-context repo="$$ctx" \ + $(_DEV_CONTAINER_ARCH_ARG) \ + $(DOCKER_BUILDX_FLAGS) \ + --load -t "$$ref" \ + "$$ctx"; \ + fi; \ + echo "$$ref" > $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/.image-ref + +# Build the linux/amd64 variant. On amd64 hosts this reuses dev-image; on +# arm64 hosts it produces a separate `:-x86_64` image so it doesn't +# clobber the native one — built locally only (CI doesn't publish the +# `-x86_64` ref), so no docker pull attempt. Buildx caches per-platform +# so repeats are cheap. +ifeq ($(_DEV_CONTAINER_PLATFORM),linux/amd64) +dev-image-x86_64: dev-image + @cp $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/.image-ref $(_DEV_CONTAINER_X86_64_IMAGE_REF_FILE) +else +dev-image-x86_64: + @$(_dev_container_stage_context); \ + $(_dev_container_compute_tag); \ + ref="$(DEV_CONTAINER_IMAGE_NAME):$$tag-x86_64"; \ + docker buildx build --platform=linux/amd64 \ + --file "$$ctx/.devcontainer/Dockerfile" \ + --build-context repo="$$ctx" \ + --build-arg ARCH=x86_64 \ + $(DOCKER_BUILDX_FLAGS) \ + --load -t "$$ref" \ + "$$ctx"; \ + echo "$$ref" > $(_DEV_CONTAINER_X86_64_IMAGE_REF_FILE) +endif + +dev-shell: dev-image + @ref=$$(cat $(DEV_CONTAINER_REPO_ROOT)/.devcontainer/.image-ref); \ + docker run --rm -it $(_DEV_CONTAINER_MOUNTS) "$$ref" /bin/sh + +endif # _DEV_CONTAINER_INSIDE diff --git a/.devcontainer/run-integration-tests.sh b/.devcontainer/run-integration-tests.sh new file mode 100755 index 0000000..cfcd025 --- /dev/null +++ b/.devcontainer/run-integration-tests.sh @@ -0,0 +1,55 @@ +#!/bin/sh +# Build mod_datadog (with RUM) and run the full integration-test suite. +# Assumes it runs inside the devcontainer image (Alpine + musl sysroot + httpd +# at /httpd/httpd-build/ + uv). +# +# If MODULE_PATH is set in the environment, the cmake build is skipped and +# pytest is pointed at the pre-built module at that path. CI uses this to +# reuse the artifact produced by the build:$ARCH job instead of rebuilding +# from source inside the test job. +set -eux + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$REPO_ROOT" + +ARCH="$(uname -m)" + +# Use a container-specific build tree to avoid colliding with host builds +# (host cmake would use Unix Makefiles; the ci-release preset uses Ninja). +BUILD_DIR="$REPO_ROOT/build-container" +DIST_DIR="$REPO_ROOT/dist-container" + +# Repo root is bind-mounted from the host; cmake complains otherwise. +git config --global --add safe.directory "$REPO_ROOT" + +if [ -z "${MODULE_PATH:-}" ]; then + cmake --preset=ci-release \ + -DHTTPD_DATADOG_ENABLE_RUM=ON \ + -DCMAKE_TOOLCHAIN_FILE="/sysroot/${ARCH}-none-linux-musl/Toolchain.cmake" \ + -B "$BUILD_DIR" . + cmake --build "$BUILD_DIR" -j + # Wipe DIST_DIR so a removed/renamed file in the build doesn't + # linger from a previous install and silently get used by tests. + rm -rf "$DIST_DIR" + cmake --install "$BUILD_DIR" --prefix "$DIST_DIR" + MODULE_PATH="$DIST_DIR/lib/mod_datadog.so" +fi + +cd test/integration-test + +# Keep the venv outside the bind-mounted repo so a host-side `uv sync` +# (darwin/arm64 wheels) can't shadow the container's Linux one. +export UV_PROJECT_ENVIRONMENT=/root/.venv-httpd-datadog-tests + +uv sync + +# Pytest's early arg parser runs BEFORE conftest registers `--bin-path`. +# If we pass `--bin-path VALUE` (space-separated), that parser treats the +# value as a positional test path and uses its parent as rootdir, which +# then prevents conftest from loading. The `=` syntax keeps the option and +# its value in a single token and avoids that race. +uv run pytest \ + --bin-path=/httpd/httpd-build/bin/apachectl \ + --module-path="$MODULE_PATH" \ + --log-dir="$REPO_ROOT/logs" \ + -v "$@" diff --git a/.github/workflows/CI_IMAGE.md b/.github/workflows/CI_IMAGE.md new file mode 100644 index 0000000..4c5b056 --- /dev/null +++ b/.github/workflows/CI_IMAGE.md @@ -0,0 +1,44 @@ +# CI image used by GitHub Actions + +The workflows in this directory pin `image:` to +`datadog/docker-library:httpd-datadog-ci-` — a public Docker Hub +mirror of the internal devcontainer image built by GitLab CI from +[`.devcontainer/Dockerfile`](../../.devcontainer/Dockerfile). The build ++ caching pipeline lives in [`.gitlab/devcontainer.yml`](../../.gitlab/devcontainer.yml); +the image-tag hash is the sha256 of the staged build context defined +by [`.devcontainer/context.files`](../../.devcontainer/context.files). + +GitHub-hosted runners can't pull from `registry.ddbuild.io`, so we +periodically copy the latest amd64 variant to Docker Hub and bump the +`image:` reference in each workflow. + +## Bumping the image + +Once GitLab CI has published a new tag (any change to a path listed in +`.devcontainer/context.files` invalidates the cache and triggers a +rebuild), mirror it from the repo root: + +```sh +make mirror-public-image +``` + +The target stages the build context, computes the same 12-char hash +GitLab uses, pulls `registry.ddbuild.io/ci/httpd-datadog/devcontainer:amd64-`, +retags it as `datadog/docker-library:httpd-datadog-ci-`, and +pushes. It then prints the exact `image:` value to copy into: + +- `dev.yml` (build + test jobs) +- `release.yml` (build job) +- `system-tests.yml` (build-artifacts job) + +## Why two image namespaces + +| Consumer | Pull from | Why | +|-----------------|----------------------------------------------------------------------------|-----| +| GitLab CI | `registry.ddbuild.io/ci/httpd-datadog/devcontainer:-` | private, fast, nydus-optimized | +| GitHub Actions | `datadog/docker-library:httpd-datadog-ci-` (mirrored from the above) | public, reachable from GitHub-hosted runners | +| Local `make` | Either of the above (pull `registry.ddbuild.io/...` if you have access; build locally otherwise) | dev convenience | + +The two mirrors stay in sync only as far as the most recent +`make mirror-public-image` run — bump the workflows promptly after +running it. diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 8fcd549..a551fa5 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -19,20 +19,12 @@ jobs: needs: format runs-on: ubuntu-22.04 container: - # See in Makefile where this image comes from. + # See .github/workflows/CI_IMAGE.md — bump via `make mirror-public-image`. image: datadog/docker-library:httpd-datadog-ci-28219c0ef3e00f1e3d5afcab61a73a5e9bd2a9b957d7545556711cce2a6262cd steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - - name: Add cloned repo as safe - run: sh -c "git config --global --add safe.directory $PWD" - - name: Init required submodules - run: git submodule update --init --depth=1 deps/dd-trace-cpp deps/nginx-datadog - - name: Configure - run: cmake --preset=ci-dev -B build . - name: Build - run: | - cmake --build build -j --verbose - cmake --install build --prefix dist + run: make ci-build PRESET=ci-dev - name: Export library uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: @@ -44,7 +36,7 @@ jobs: needs: build runs-on: ubuntu-22.04 container: - # See in Makefile where this image comes from. + # See .github/workflows/CI_IMAGE.md — bump via `make mirror-public-image`. image: datadog/docker-library:httpd-datadog-ci-28219c0ef3e00f1e3d5afcab61a73a5e9bd2a9b957d7545556711cce2a6262cd env: DD_ENV: ci diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7a7539b..6514dd7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,20 +5,12 @@ jobs: build: runs-on: ubuntu-22.04 container: - # See in Makefile where this image comes from. + # See .github/workflows/CI_IMAGE.md — bump via `make mirror-public-image`. image: datadog/docker-library:httpd-datadog-ci-28219c0ef3e00f1e3d5afcab61a73a5e9bd2a9b957d7545556711cce2a6262cd steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - - name: Add cloned repo as safe - run: sh -c "git config --global --add safe.directory $PWD" - - name: Init required submodules - run: git submodule update --init --depth=1 deps/dd-trace-cpp deps/nginx-datadog - - name: Configure - run: cmake --preset ci-release -B build . - name: Build - run: | - cmake --build build -j --verbose - cmake --install build --prefix dist + run: make ci-build PRESET=ci-release - name: Export library uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 72414ee..dbde772 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -19,20 +19,12 @@ jobs: build-artifacts: runs-on: ubuntu-22.04 container: - # See in Makefile where this image comes from. + # See .github/workflows/CI_IMAGE.md — bump via `make mirror-public-image`. image: datadog/docker-library:httpd-datadog-ci-28219c0ef3e00f1e3d5afcab61a73a5e9bd2a9b957d7545556711cce2a6262cd steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - - name: Add cloned repo as safe - run: sh -c "git config --global --add safe.directory $PWD" - - name: Init required submodules - run: git submodule update --init --depth=1 deps/dd-trace-cpp deps/nginx-datadog - - name: Configure - run: cmake --preset=ci-dev -B build . - name: Build - run: | - cmake --build build -j --verbose - cmake --install build --prefix dist + run: make ci-build PRESET=ci-dev - name: Export library uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: diff --git a/.gitignore b/.gitignore index 75feb22..cb252be 100644 --- a/.gitignore +++ b/.gitignore @@ -23,10 +23,13 @@ compile_commands.json __pycache__/ +build-container/ build-rum/ build/ +dist-container/ httpd-*/ httpd/ +/logs/ venv/ test/integration-test/.coverage @@ -37,3 +40,6 @@ test/integration-test/htmlcov/ test/integration-test/log-*/ test/integration-test/logs/ test/integration-test/uv.lock +.devcontainer/.staged/ +.devcontainer/.image-ref +.devcontainer/.image-ref-x86_64 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4a328df..9e9b3b6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,15 @@ +include: + - local: .gitlab/devcontainer.yml + variables: - CI_DOCKER_IMAGE_BASE: registry.ddbuild.io/ci/httpd-datadog + # Inputs to .gitlab/devcontainer.yml (the dd-repo-tools shared + # devcontainer template). DEVCONTAINER_PER_ARCH=true publishes + # $IMAGE:amd64-$TAG and $IMAGE:arm64-$TAG; the downstream build / test + # jobs below then pull the matching arch and run on native runners + # (`tags: ["arch:$ARCH"]`). The cmake compile + integration-test suite + # is too slow under qemu to run any other way. + DEVCONTAINER_REPO_NAME: httpd-datadog + DEVCONTAINER_PER_ARCH: "true" GIT_SUBMODULE_STRATEGY: recursive GIT_SUBMODULE_DEPTH: 1 GIT_CONFIG_COUNT: 1 @@ -12,7 +22,6 @@ default: aud: image-integrity stages: - - ci-image - build - test - aws @@ -20,7 +29,9 @@ stages: # Template jobs (hidden, reusable) .build-template: stage: build - image: $CI_DOCKER_IMAGE_BASE:$CI_IMAGE_HASH + needs: + - job: devcontainer_image + artifacts: true script: - git config --global --add safe.directory $PWD - > @@ -42,97 +53,16 @@ stages: .test-rum-template: stage: test - image: $CI_DOCKER_IMAGE_BASE:$CI_IMAGE_HASH + # Same `--flag=VALUE` reasoning as run-integration-tests.sh: pytest's early + # arg parser runs before conftest registers these custom options, so the + # space-separated form gets read as a positional test path and skips + # conftest. script: - - cd $CI_PROJECT_DIR/test/integration-test && uv sync && uv run pytest scenarios --module-path $CI_PROJECT_DIR/artifacts/$ARCH/mod_datadog.so --bin-path /httpd/httpd-build/bin/apachectl --log-dir $CI_PROJECT_DIR/logs -m requires_rum -v - -build-ci-image: - stage: ci-image - tags: ["arch:$ARCH"] - image: registry.ddbuild.io/images/docker:27.3.1 - parallel: - matrix: - - ARCH: amd64 - TOOLCHAIN_ARCH: x86_64 - - ARCH: arm64 - TOOLCHAIN_ARCH: aarch64 - script: - - | - FILES=$(find .gitlab/Dockerfile deps/nginx-datadog/build_env/ scripts/setup-httpd.py -type f | sort) - HASH=$(for f in $FILES; do echo "--- $f ---"; cat "$f"; done | sha256sum | cut -d ' ' -f 1) - - echo "CI_IMAGE_HASH=$HASH" >> ci-image.env - - IMAGE_TAG="$CI_DOCKER_IMAGE_BASE/$ARCH:$HASH" - - | - if docker buildx imagetools inspect "$IMAGE_TAG" > /dev/null 2>&1; then - echo "Image $IMAGE_TAG already exists. Skipping build." - else - docker buildx build \ - --tag $IMAGE_TAG \ - --platform linux/$ARCH \ - --build-arg ARCH=$TOOLCHAIN_ARCH \ - --push \ - --file .gitlab/Dockerfile \ - . - echo "Image $IMAGE_TAG built for $ARCH." - fi - artifacts: - reports: - dotenv: ci-image.env - -create-ci-manifest: - stage: ci-image - needs: - - build-ci-image - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:27.3.1 - script: - - MANIFEST_TAG="$CI_DOCKER_IMAGE_BASE:$CI_IMAGE_HASH" - - | - if docker buildx imagetools inspect "$MANIFEST_TAG" > /dev/null 2>&1; then - echo "Multi-arch manifest $MANIFEST_TAG already exists. Skipping." - else - docker buildx imagetools create \ - --tag "$MANIFEST_TAG" \ - "$CI_DOCKER_IMAGE_BASE/amd64:$CI_IMAGE_HASH" \ - "$CI_DOCKER_IMAGE_BASE/arm64:$CI_IMAGE_HASH" - fi - -nydusify-ci-image: - stage: ci-image - needs: - - build-ci-image - - create-ci-manifest - allow_failure: true - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/nydus:v98111448-6d7c7d0-v2.3.8-dd - variables: - DDSIGN_SKIP_SIGNING: "true" - script: - - | - IMAGE="$CI_DOCKER_IMAGE_BASE:$CI_IMAGE_HASH" - REPO="${IMAGE%:*}" - - # Resolve a platform manifest digest from the multi-arch index - DIGEST=$(crane manifest "$IMAGE" 2>/dev/null | jq -r '(.manifests // [])[0].digest // empty' 2>/dev/null) - if [ -n "$DIGEST" ]; then - MANIFEST=$(crane manifest "${REPO}@${DIGEST}" 2>/dev/null) - else - MANIFEST=$(crane manifest "$IMAGE" 2>/dev/null) - fi - - # Check if the platform manifest already contains nydus layers - if echo "$MANIFEST" | jq -e '[.layers[].mediaType] | any(contains("nydus"))' > /dev/null 2>&1; then - echo "$IMAGE is already nydusified. Skipping." - else - echo "Converting $IMAGE with nydusify..." - nydus-convert "$IMAGE" "$IMAGE" - fi + - cd $CI_PROJECT_DIR/test/integration-test && uv sync && uv run pytest scenarios --module-path=$CI_PROJECT_DIR/artifacts/$ARCH/mod_datadog.so --bin-path=/httpd/httpd-build/bin/apachectl --log-dir=$CI_PROJECT_DIR/logs -m requires_rum -v build:amd64: extends: .build-template - needs: - - build-ci-image - - create-ci-manifest + image: $DEVCONTAINER_IMAGE_AMD64 variables: ARCH: amd64 TOOLCHAIN_ARCH: x86_64 @@ -140,9 +70,7 @@ build:amd64: build:arm64: extends: .build-template - needs: - - build-ci-image - - create-ci-manifest + image: $DEVCONTAINER_IMAGE_ARM64 variables: ARCH: arm64 TOOLCHAIN_ARCH: aarch64 @@ -150,18 +78,24 @@ build:arm64: test-rum:amd64: extends: .test-rum-template + image: $DEVCONTAINER_IMAGE_AMD64 needs: - - build-ci-image + - job: devcontainer_image + artifacts: true - job: "build:amd64" + artifacts: true variables: ARCH: amd64 tags: ["arch:amd64"] test-rum:arm64: extends: .test-rum-template + image: $DEVCONTAINER_IMAGE_ARM64 needs: - - build-ci-image + - job: devcontainer_image + artifacts: true - job: "build:arm64" + artifacts: true variables: ARCH: arm64 tags: ["arch:arm64"] diff --git a/.gitlab/devcontainer.yml b/.gitlab/devcontainer.yml new file mode 100644 index 0000000..9a55880 --- /dev/null +++ b/.gitlab/devcontainer.yml @@ -0,0 +1,124 @@ +# DO NOT EDIT. Synced from DataDog/dd-repo-tools by the +# dd-repo-tools devcontainer-bundle campaigner into +# `.gitlab/devcontainer.yml` — edits here will be overwritten. +# +# Shared devcontainer pipeline. One job does everything: stage the +# context, compute the tag, check the cache, build + nydusify on miss, +# emit ci-image.env for downstream jobs. +# +# Consumer repo wires this in via: +# variables: +# DEVCONTAINER_REPO_NAME: my-repo +# # optional: +# # DEVCONTAINER_IMAGE_NAME: registry.ddbuild.io/ci/my-repo # flat path +# # DEVCONTAINER_PER_ARCH: "true" # arch matrix +# include: +# - local: .gitlab/devcontainer.yml +# +# DEVCONTAINER_PER_ARCH=true builds two single-arch refs ($IMAGE:amd64-$TAG +# and $IMAGE:arm64-$TAG) instead of one multi-arch manifest. Downstream +# jobs pulling the matching ref can then run on a native-arch runner, +# avoiding qemu emulation at consume time. Both builds still happen +# sequentially in this single .pre job pinned to arch:amd64 — moving the +# build itself to a parallel:matrix on native runners is future work. +# Default off because most repos build fast enough that the single +# multi-arch buildx run is the cheaper option. +# +# Per-repo inputs live in: +# .devcontainer/Dockerfile the dockerfile +# .devcontainer/context.files rsync --files-from list (one path per line; +# '#'/blank lines are comments, stripped +# before being handed to rsync) +# .devcontainer/context.filter optional rsync filter rules + +variables: + DEVCONTAINER_IMAGE_NAME: registry.ddbuild.io/ci/${DEVCONTAINER_REPO_NAME}/devcontainer + DEVCONTAINER_PER_ARCH: "" + # Bootstrap image used by the devcontainer_image job. The signed-retag + # flow (DEVCONTAINER_BOOTSTRAP_IMAGE_TAG) is wired up but the producing + # job is on hold along with template publishing. + DEVCONTAINER_BOOTSTRAP_IMAGE_BASE: registry.ddbuild.io/ci/dd-repo-tools/devcontainer-bootstrap + DEVCONTAINER_BOOTSTRAP_IMAGE_TAG: "" + DEVCONTAINER_BOOTSTRAP_IMAGE: registry.ddbuild.io/images/nydus:v2.4.0-dd.3 + +devcontainer_image: + stage: .pre + image: $DEVCONTAINER_BOOTSTRAP_IMAGE + needs: [] + tags: ["arch:amd64"] + # Compose the bootstrap image from BASE:TAG when a tag is pinned (the + # post-`publish-devcontainer-bootstrap-image` workflow), otherwise fall + # back to the upstream nydus image defined in `variables:` above. + rules: + - if: '$DEVCONTAINER_BOOTSTRAP_IMAGE_TAG' + variables: + DEVCONTAINER_BOOTSTRAP_IMAGE: $DEVCONTAINER_BOOTSTRAP_IMAGE_BASE:$DEVCONTAINER_BOOTSTRAP_IMAGE_TAG + - when: always + script: + - | + set -euo pipefail + + # Bootstrap image is minimal — install make (and rsync, which the + # staging step needs) on demand. Each `command -v` guards a single + # apt-get so a re-run on an image that already has them is a no-op. + missing="" + command -v make >/dev/null || missing="$missing make" + command -v rsync >/dev/null || missing="$missing rsync" + if [ -n "$missing" ]; then + apt-get update -qq + # shellcheck disable=SC2086 + apt-get install -y -qq --no-install-recommends $missing + fi + + # Stage + hash via devcontainer.mk so CI and local `make` use the + # same code. `-f` is explicit so we don't depend on the consumer's + # root Makefile including the fragment. .devcontainer-stage-context + # writes to .devcontainer/.staged/; .devcontainer-image-hash reads + # from the same path and prints the 12-char tag. + MK="-f .devcontainer/devcontainer.mk" + # shellcheck disable=SC2086 + make $MK .devcontainer-stage-context + ctx=.devcontainer/.staged + # shellcheck disable=SC2086 + TAG=$(make $MK -s .devcontainer-image-hash) + echo "Devcontainer tag: $TAG" + + # ---- build-or-skip per arch ---- + build_and_nydusify() { + local ref=$1 platform=$2 arch_arg=$3 + echo "Building $ref ..." + # shellcheck disable=SC2086 + docker buildx build \ + --platform "$platform" \ + --file "$ctx/.devcontainer/Dockerfile" \ + --build-context "repo=$ctx" \ + $arch_arg \ + --push -t "$ref" \ + "$ctx" + nydus-convert "$ref" "$ref" + } + + if [ -n "$DEVCONTAINER_PER_ARCH" ]; then + AMD_REF="${DEVCONTAINER_IMAGE_NAME}:amd64-${TAG}" + ARM_REF="${DEVCONTAINER_IMAGE_NAME}:arm64-${TAG}" + crane manifest "$AMD_REF" >/dev/null 2>&1 \ + || build_and_nydusify "$AMD_REF" linux/amd64 "--build-arg ARCH=x86_64" + crane manifest "$ARM_REF" >/dev/null 2>&1 \ + || build_and_nydusify "$ARM_REF" linux/arm64 "--build-arg ARCH=aarch64" + { + echo "DEVCONTAINER_IMAGE_AMD64=$AMD_REF" + echo "DEVCONTAINER_IMAGE_ARM64=$ARM_REF" + echo "CI_IMAGE_HASH=$TAG" + } > ci-image.env + else + REF="${DEVCONTAINER_IMAGE_NAME}:${TAG}" + crane manifest "$REF" >/dev/null 2>&1 \ + || build_and_nydusify "$REF" linux/amd64,linux/arm64 "" + { + echo "DEVCONTAINER_IMAGE=$REF" + echo "CI_IMAGE_HASH=$TAG" + } > ci-image.env + fi + artifacts: + reports: + dotenv: ci-image.env diff --git a/.gitmodules b/.gitmodules index 52a8a84..52183b5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,10 +1,10 @@ [submodule "deps/dd-trace-cpp"] path = deps/dd-trace-cpp - url = ../dd-trace-cpp.git + url = https://github.com/DataDog/dd-trace-cpp.git [submodule "deps/nginx-datadog"] path = deps/nginx-datadog - url = ../httpd-datadog.git + url = https://github.com/DataDog/httpd-datadog.git branch = mirror-nginx-datadog [submodule "deps/inject-browser-sdk"] path = deps/inject-browser-sdk - url = ../inject-browser-sdk.git + url = https://github.com/DataDog/inject-browser-sdk.git diff --git a/Makefile b/Makefile index 4360495..63610c0 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,60 @@ -# The CI image used for some GitHub jobs is built by the GitLab build-ci-image job. -# The hash is computed by this GitLab job from the files used to build the image. +# Pin the image name so it doesn't drift with the worktree directory name +# (the upstream default is derived from $(notdir $(CURDIR)) — fine for a +# canonical clone called `httpd-datadog`, wrong for any feature-branch +# worktree). Matches DEVCONTAINER_IMAGE_NAME in .gitlab-ci.yml. +DEV_CONTAINER_IMAGE_NAME := registry.ddbuild.io/ci/httpd-datadog/devcontainer + +# The devcontainer Dockerfile pulls toolchain files from the +# nginx-datadog submodule's build_env/. Without this guard the staging +# step silently produces a tree missing those files and the resulting +# tag mismatches what CI computes — fail loudly instead. # -# Whenever this image needs to be updated, one should: -# - Get the hash from a run of the GitLab build-ci-image job. -# - Copy this hash here, and in the GitHub workflow files. -# - Run: make replicate-ci-image-for-github. +# Skipped for `ci-build`, which runs its own `git submodule update` +# inline (the GitHub workflows checkout without submodules and init +# them as part of the build). All other targets (dev-image, +# test-integration, mirror-public-image) genuinely need build_env +# already present and keep the guard. +ifneq ($(MAKECMDGOALS),ci-build) +DEV_CONTAINER_REQUIRED_PATHS := deps/nginx-datadog/build_env/Toolchain.cmake.x86_64 +endif + +include .devcontainer/devcontainer.mk + +.PHONY: test-integration +test-integration: dev-image + $(IN_DEVCONTAINER) .devcontainer/run-integration-tests.sh -CI_DOCKER_IMAGE_HASH ?= 28219c0ef3e00f1e3d5afcab61a73a5e9bd2a9b957d7545556711cce2a6262cd -CI_IMAGE_FROM_GITLAB ?= registry.ddbuild.io/ci/httpd-datadog/amd64:$(CI_DOCKER_IMAGE_HASH) -CI_IMAGE_IN_PUBLIC_REPO_FOR_GITHUB ?= datadog/docker-library:httpd-datadog-ci-$(CI_DOCKER_IMAGE_HASH) +# One-shot CI build: trust the workdir, init the submodules cmake +# actually consumes, configure/build/install. Used by every job in +# .github/workflows/ that produces mod_datadog.so. Switch presets via +# PRESET=ci-release. inject-browser-sdk is omitted from the submodule +# list because RUM is off by default (HTTPD_DATADOG_ENABLE_RUM=OFF). +PRESET ?= ci-dev +.PHONY: ci-build +ci-build: + git config --global --add safe.directory $(CURDIR) + git submodule update --init --depth=1 deps/dd-trace-cpp deps/nginx-datadog + cmake --preset=$(PRESET) -B build . + cmake --build build -j --verbose + cmake --install build --prefix dist -.PHONY: replicate-ci-image-for-github -replicate-ci-image-for-github: - docker pull $(CI_IMAGE_FROM_GITLAB) - docker tag $(CI_IMAGE_FROM_GITLAB) $(CI_IMAGE_IN_PUBLIC_REPO_FOR_GITHUB) - docker push $(CI_IMAGE_IN_PUBLIC_REPO_FOR_GITHUB) +# GitHub-hosted runners can't pull from registry.ddbuild.io; the workflows +# in .github/workflows/ pin to a public Docker Hub mirror at +# datadog/docker-library:httpd-datadog-ci-. After a new tag is built +# by .gitlab/devcontainer.yml (any change to .devcontainer/context.files +# inputs invalidates the cache), run this target to copy the amd64 variant +# to Docker Hub and bump the workflows. +# See .github/workflows/CI_IMAGE.md for the bump procedure. +.PHONY: mirror-public-image +mirror-public-image: + @$(MAKE) -s -f .devcontainer/devcontainer.mk .devcontainer-stage-context + @hash=$$($(MAKE) -s -f .devcontainer/devcontainer.mk .devcontainer-image-hash); \ + src="$(DEV_CONTAINER_IMAGE_NAME):amd64-$$hash"; \ + public="datadog/docker-library:httpd-datadog-ci-$$hash"; \ + echo "Mirroring $$src -> $$public"; \ + docker pull --platform linux/amd64 "$$src"; \ + docker tag "$$src" "$$public"; \ + docker push "$$public"; \ + echo ""; \ + echo "Update image: in .github/workflows/{dev,release,system-tests}.yml to:"; \ + echo " $$public"