From dfc187e0436742dbf88469c97d42f496d56bdaa6 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 17:06:32 -0500 Subject: [PATCH 1/3] ci: run Codex Linux arm64 releases natively The release workflow cross-compiles Linux arm64 on an x64 runner and skips the smoke test for arm64 POSIX bundles. That prevents the workflow from executing the Linux artifact it just produced. Run Linux arm64 on GitHub's arm64 runner and install native development packages rather than configuring a foreign dpkg architecture. All matrix entries can then run the existing distribution smoke test. --- .github/workflows/codex-release.yml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/codex-release.yml b/.github/workflows/codex-release.yml index 98ffccc6e23e19..8e27edf52fc347 100644 --- a/.github/workflows/codex-release.yml +++ b/.github/workflows/codex-release.yml @@ -79,8 +79,9 @@ jobs: file_pattern: Mach-O 64-bit executable x86_64 has_gcm: false max_tar_bytes: 67108864 + # Keep arm64 builds native so release smoke tests can execute them. - name: Linux arm64 - os: ubuntu-22.04 + os: ubuntu-22.04-arm target_platform: ubuntu asset_platform: ubuntu arch: arm64 @@ -195,19 +196,13 @@ jobs: - name: Install Linux arm64 build dependencies if: matrix.target_platform == 'ubuntu' && matrix.arch == 'arm64' run: | - sudo sed -i "s/^deb/deb [arch=amd64,i386]/g" /etc/apt/sources.list - release="$(lsb_release -s -c)" - echo "deb [arch=arm64,armhf] http://azure.ports.ubuntu.com/ ${release} main universe multiverse restricted" | sudo tee -a /etc/apt/sources.list - echo "deb [arch=arm64,armhf] http://azure.ports.ubuntu.com/ ${release}-updates main universe multiverse restricted" | sudo tee -a /etc/apt/sources.list - sudo dpkg --add-architecture arm64 - sudo apt-get update sudo apt-get install -y \ binutils-aarch64-linux-gnu \ gcc-aarch64-linux-gnu \ - libcurl4-gnutls-dev:arm64 \ - libexpat1-dev:arm64 \ - libssl-dev:arm64 \ - zlib1g-dev:arm64 + libcurl4-gnutls-dev \ + libexpat1-dev \ + libssl-dev \ + zlib1g-dev # Dugite Native currently pins MinGit 2.53. Keep its build script and # dependency schema, but match the runtime to the Git series we compile. @@ -341,7 +336,6 @@ jobs: strings "$GIT_BINARY" | grep -F "$GITHUB_SHA" - name: Smoke-test the native distribution - if: matrix.arch == 'x64' || matrix.target_platform == 'win32' shell: bash env: TARGET_PLATFORM: ${{ matrix.target_platform }} From 1f8a4fd9cd6072a886cff6784c33457cb84e070f Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 17:10:52 -0500 Subject: [PATCH 2/3] ci: build Codex Git releases with LTO Codex consumes Git release artifacts built with the Makefile's default -O2 flags. The release job compiles each artifact without link-time optimization. Add a release-only config.mak.openai and copy it into Git's ignored config.mak slot before building. Use thin LTO for Clang targets and automatic LTO for GCC targets, then check GIT-CFLAGS records the selected flag in every distribution job. Keeping the setting in config.mak.openai avoids carrying release-only policy in the upstream Makefile. --- .github/workflows/codex-release.yml | 15 +++++++++++++++ config.mak.openai | 10 ++++++++++ 2 files changed, 25 insertions(+) create mode 100644 config.mak.openai diff --git a/.github/workflows/codex-release.yml b/.github/workflows/codex-release.yml index 8e27edf52fc347..54b8e94e12b2b9 100644 --- a/.github/workflows/codex-release.yml +++ b/.github/workflows/codex-release.yml @@ -69,6 +69,7 @@ jobs: binary: /tmp/build/git/bin/git file_pattern: Mach-O 64-bit executable arm64 has_gcm: false + lto: thin max_tar_bytes: 67108864 - name: macOS x64 os: macos-15-intel @@ -78,6 +79,7 @@ jobs: binary: /tmp/build/git/bin/git file_pattern: Mach-O 64-bit executable x86_64 has_gcm: false + lto: thin max_tar_bytes: 67108864 # Keep arm64 builds native so release smoke tests can execute them. - name: Linux arm64 @@ -88,6 +90,7 @@ jobs: binary: /tmp/build/git/bin/git file_pattern: ELF 64-bit.*ARM aarch64 has_gcm: false + lto: auto max_tar_bytes: 67108864 - name: Linux x64 os: ubuntu-22.04 @@ -97,6 +100,7 @@ jobs: binary: /tmp/build/git/bin/git file_pattern: ELF 64-bit.*x86-64 has_gcm: false + lto: auto max_tar_bytes: 67108864 - name: Windows arm64 os: windows-11-arm @@ -106,6 +110,7 @@ jobs: binary: /tmp/build/git/clangarm64/bin/git.exe file_pattern: PE32\+.*ARM64 has_gcm: true + lto: thin max_tar_bytes: 134217728 sdk_arch: aarch64 sdk_flavor: full @@ -122,6 +127,7 @@ jobs: binary: /tmp/build/git/mingw64/bin/git.exe file_pattern: PE32\+.*x86-64 has_gcm: true + lto: auto max_tar_bytes: 134217728 sdk_arch: x86_64 sdk_flavor: full @@ -162,6 +168,11 @@ jobs: -c 'user.email=41898282+github-actions[bot]@users.noreply.github.com' \ tag -a "$VERSION" -m "$VERSION" + - name: Install OpenAI build configuration + shell: bash + working-directory: dugite-native/git + run: cp config.mak.openai config.mak + # Match Dugite Native's compatibility choice for its macOS x64 build. - name: Select Xcode 16.4 if: matrix.target_platform == 'macOS' && matrix.arch == 'x64' @@ -257,6 +268,7 @@ jobs: working-directory: dugite-native env: NO_RUST: 1 + OPENAI_LTO: ${{ matrix.lto }} TARGET_PLATFORM: ${{ matrix.target_platform }} TARGET_ARCH: ${{ matrix.arch }} run: | @@ -282,6 +294,7 @@ jobs: working-directory: dugite-native/git env: MINGW_DIR: ${{ matrix.mingw_dir }} + OPENAI_LTO: ${{ matrix.lto }} run: | set -euo pipefail @@ -307,6 +320,7 @@ jobs: GIT_BINARY: ${{ matrix.binary }} FILE_PATTERN: ${{ matrix.file_pattern }} HAS_GCM: ${{ matrix.has_gcm }} + LTO: ${{ matrix.lto }} run: | set -euo pipefail if test "$TARGET_PLATFORM" = win32 @@ -334,6 +348,7 @@ jobs: file "$GIT_BINARY" | tee /tmp/git-file-type grep -E "$FILE_PATTERN" /tmp/git-file-type strings "$GIT_BINARY" | grep -F "$GITHUB_SHA" + grep -F -- "-flto=$LTO" dugite-native/git/GIT-CFLAGS - name: Smoke-test the native distribution shell: bash diff --git a/config.mak.openai b/config.mak.openai new file mode 100644 index 00000000000000..d75c9aee7d4903 --- /dev/null +++ b/config.mak.openai @@ -0,0 +1,10 @@ +# OpenAI release build settings. +# +# The Codex release workflow copies this file to config.mak before +# building. Keep release-only optimizations here instead of in the +# upstream Makefile. + +ifdef OPENAI_LTO +CFLAGS_APPEND += -flto=$(OPENAI_LTO) +LDFLAGS_APPEND += -flto=$(OPENAI_LTO) +endif From ba099830983b25f519048c1c8b35217b16a62fe5 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 17:12:16 -0500 Subject: [PATCH 3/3] ci: train Codex Git releases with PGO LTO can optimize across translation units, but the release job has no execution profile for the status, diff, clone, fetch, and repack paths Codex invokes frequently. Git's built-in profile target runs the 1,048-script test suite serially. That is too expensive for every release target and weights test-harness paths more heavily than the local workload. Extend config.mak.openai with GCC and LLVM profile modes. Gate GIT-CFLAGS on an instrumented build, run a short offline trainer, merge LLVM raw profiles when needed, and rebuild with profile-use flags. Each matrix entry runs on its target architecture, so it can execute the instrumented binary. Check that final GIT-CFLAGS includes a profile-use flag and increase the timeout for the second compilation pass. The focused trainer took about 30 seconds locally; the full macOS build/install validation completed with thin LTO and LLVM profile-use enabled. --- .github/workflows/codex-pgo-training.sh | 60 +++++++++++++++++++++++ .github/workflows/codex-release.yml | 32 +++++++++++-- config.mak.openai | 64 +++++++++++++++++++++++++ 3 files changed, 151 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/codex-pgo-training.sh diff --git a/.github/workflows/codex-pgo-training.sh b/.github/workflows/codex-pgo-training.sh new file mode 100644 index 00000000000000..070d8b54cae486 --- /dev/null +++ b/.github/workflows/codex-pgo-training.sh @@ -0,0 +1,60 @@ +#!/bin/sh + +# Keep this workload short and biased toward the local Git operations Codex +# invokes frequently. The full Git test suite is too slow for each release +# target and would weight test-harness paths more heavily than status, diff, +# clone, fetch, and repository maintenance. + +set -eu + +git_bin="$PWD/bin-wrappers/git" +training_dir=$(mktemp -d "${TMPDIR:-/tmp}/codex-git-pgo.XXXXXX") +repo="$training_dir/repo" +clone="$training_dir/clone" + +cleanup () { + rm -rf "$training_dir" +} +trap cleanup EXIT HUP INT TERM + +mkdir -p "$training_dir/home" +export HOME="$training_dir/home" +export GIT_CONFIG_NOSYSTEM=1 +export GIT_TERMINAL_PROMPT=0 + +"$git_bin" clone --quiet --no-local "$PWD" "$repo" +"$git_bin" -C "$repo" config user.name "Codex Git PGO" +"$git_bin" -C "$repo" config user.email "codex-git-pgo@openai.com" + +i=0 +while test "$i" -lt 256 +do + dir="$repo/training/$((i % 16))" + mkdir -p "$dir" + printf '%s\n' "$i" >"$dir/file-$i" + i=$((i + 1)) +done + +"$git_bin" -C "$repo" status --porcelain=v2 --branch >/dev/null +"$git_bin" -C "$repo" status --porcelain=v2 --branch --untracked-files=all >/dev/null +"$git_bin" -C "$repo" ls-files --others --exclude-standard >/dev/null +"$git_bin" -C "$repo" add training +"$git_bin" -C "$repo" diff --cached --stat >/dev/null +"$git_bin" -C "$repo" commit --quiet -m "add training files" + +printf 'changed\n' >>"$repo/training/0/file-0" +rm "$repo/training/1/file-1" +mkdir -p "$repo/untracked" +printf 'new\n' >"$repo/untracked/file" + +"$git_bin" -C "$repo" status --porcelain=v2 --branch >/dev/null +"$git_bin" -C "$repo" diff --stat >/dev/null +"$git_bin" -C "$repo" diff --name-status >/dev/null +"$git_bin" -C "$repo" ls-files --stage >/dev/null +"$git_bin" -C "$repo" log --oneline --decorate -20 >/dev/null +"$git_bin" -C "$repo" rev-list --objects --all >/dev/null +"$git_bin" -C "$repo" for-each-ref --format='%(refname) %(objectname)' >/dev/null +"$git_bin" -C "$repo" repack -ad +"$git_bin" clone --quiet --no-local "$repo" "$clone" +"$git_bin" -C "$clone" status --porcelain=v2 --branch >/dev/null +"$git_bin" -C "$clone" fetch --quiet "$repo" diff --git a/.github/workflows/codex-release.yml b/.github/workflows/codex-release.yml index 54b8e94e12b2b9..bf0ea496175a04 100644 --- a/.github/workflows/codex-release.yml +++ b/.github/workflows/codex-release.yml @@ -56,7 +56,7 @@ jobs: name: ${{ matrix.name }} needs: version runs-on: ${{ matrix.os }} - timeout-minutes: 30 + timeout-minutes: 60 strategy: fail-fast: false matrix: @@ -70,6 +70,8 @@ jobs: file_pattern: Mach-O 64-bit executable arm64 has_gcm: false lto: thin + profile_format: LLVM + llvm_profdata: xcrun llvm-profdata max_tar_bytes: 67108864 - name: macOS x64 os: macos-15-intel @@ -80,6 +82,8 @@ jobs: file_pattern: Mach-O 64-bit executable x86_64 has_gcm: false lto: thin + profile_format: LLVM + llvm_profdata: xcrun llvm-profdata max_tar_bytes: 67108864 # Keep arm64 builds native so release smoke tests can execute them. - name: Linux arm64 @@ -91,6 +95,7 @@ jobs: file_pattern: ELF 64-bit.*ARM aarch64 has_gcm: false lto: auto + profile_format: GCC max_tar_bytes: 67108864 - name: Linux x64 os: ubuntu-22.04 @@ -101,6 +106,7 @@ jobs: file_pattern: ELF 64-bit.*x86-64 has_gcm: false lto: auto + profile_format: GCC max_tar_bytes: 67108864 - name: Windows arm64 os: windows-11-arm @@ -111,6 +117,8 @@ jobs: file_pattern: PE32\+.*ARM64 has_gcm: true lto: thin + profile_format: LLVM + llvm_profdata: llvm-profdata max_tar_bytes: 134217728 sdk_arch: aarch64 sdk_flavor: full @@ -128,6 +136,7 @@ jobs: file_pattern: PE32\+.*x86-64 has_gcm: true lto: auto + profile_format: GCC max_tar_bytes: 134217728 sdk_arch: x86_64 sdk_flavor: full @@ -261,14 +270,18 @@ jobs: dependencies.json >"$updated" mv "$updated" dependencies.json - # Dugite cross-compiles several targets. Keep Git's optional Rust - # library disabled until its Makefile can direct Cargo at those targets. + # Build an instrumented Git, run a representative local workload, then + # rebuild with its profile. Keep Git's optional Rust library disabled + # until its Makefile can direct Cargo at these targets. - name: Build the Dugite Native distribution shell: bash working-directory: dugite-native env: NO_RUST: 1 + OPENAI_LLVM_PROFDATA: ${{ matrix.llvm_profdata }} OPENAI_LTO: ${{ matrix.lto }} + OPENAI_PROFILE: BUILD + OPENAI_PROFILE_FORMAT: ${{ matrix.profile_format }} TARGET_PLATFORM: ${{ matrix.target_platform }} TARGET_ARCH: ${{ matrix.arch }} run: | @@ -294,7 +307,9 @@ jobs: working-directory: dugite-native/git env: MINGW_DIR: ${{ matrix.mingw_dir }} + OPENAI_LLVM_PROFDATA: ${{ matrix.llvm_profdata }} OPENAI_LTO: ${{ matrix.lto }} + OPENAI_PROFILE_FORMAT: ${{ matrix.profile_format }} run: | set -euo pipefail @@ -309,8 +324,8 @@ jobs: SKIP_DASHED_BUILT_INS=YesPlease ) jobs="${NUMBER_OF_PROCESSORS:-2}" - make -j"$jobs" "${make_args[@]}" all - make "${make_args[@]}" DESTDIR=/tmp/build/git strip install + make -j"$jobs" "${make_args[@]}" OPENAI_PROFILE=BUILD all + make "${make_args[@]}" OPENAI_PROFILE=USE DESTDIR=/tmp/build/git strip install - name: Verify distribution layout and provenance shell: bash @@ -321,6 +336,7 @@ jobs: FILE_PATTERN: ${{ matrix.file_pattern }} HAS_GCM: ${{ matrix.has_gcm }} LTO: ${{ matrix.lto }} + PROFILE_FORMAT: ${{ matrix.profile_format }} run: | set -euo pipefail if test "$TARGET_PLATFORM" = win32 @@ -349,6 +365,12 @@ jobs: grep -E "$FILE_PATTERN" /tmp/git-file-type strings "$GIT_BINARY" | grep -F "$GITHUB_SHA" grep -F -- "-flto=$LTO" dugite-native/git/GIT-CFLAGS + if test "$PROFILE_FORMAT" = LLVM + then + grep -F -- "-fprofile-instr-use=" dugite-native/git/GIT-CFLAGS + else + grep -F -- "-fprofile-use=" dugite-native/git/GIT-CFLAGS + fi - name: Smoke-test the native distribution shell: bash diff --git a/config.mak.openai b/config.mak.openai index d75c9aee7d4903..0ae133c8bb2ecf 100644 --- a/config.mak.openai +++ b/config.mak.openai @@ -4,7 +4,71 @@ # building. Keep release-only optimizations here instead of in the # upstream Makefile. +OPENAI_PROFILE_DIR := $(CURDIR) +OPENAI_PROFILE_RAW := $(OPENAI_PROFILE_DIR)/default_%m_%p.profraw +OPENAI_PROFILE_DATA := $(OPENAI_PROFILE_DIR)/default.profdata +OPENAI_PROFILE_TRAINING ?= .github/workflows/codex-pgo-training.sh +OPENAI_LLVM_PROFDATA ?= llvm-profdata + ifdef OPENAI_LTO CFLAGS_APPEND += -flto=$(OPENAI_LTO) LDFLAGS_APPEND += -flto=$(OPENAI_LTO) endif + +ifeq ("$(OPENAI_PROFILE_FORMAT)","LLVM") +ifeq ("$(OPENAI_PROFILE)","GEN") + BASIC_CFLAGS += -fprofile-instr-generate=$(OPENAI_PROFILE_RAW) + BASIC_CFLAGS += -DNO_NORETURN=1 + export CCACHE_DISABLE = t + V = 1 +else +ifneq ("$(OPENAI_PROFILE)","") + BASIC_CFLAGS += -fprofile-instr-use=$(OPENAI_PROFILE_DATA) + BASIC_CFLAGS += -Wno-profile-instr-unprofiled -DNO_NORETURN=1 + export CCACHE_DISABLE = t + V = 1 +endif +endif +else +ifeq ("$(OPENAI_PROFILE)","GEN") + BASIC_CFLAGS += -fprofile-generate=$(OPENAI_PROFILE_DIR) + BASIC_CFLAGS += -DNO_NORETURN=1 + EXTLIBS += -lgcov + export CCACHE_DISABLE = t + V = 1 +else +ifneq ("$(OPENAI_PROFILE)","") + BASIC_CFLAGS += -fprofile-use=$(OPENAI_PROFILE_DIR) + BASIC_CFLAGS += -fprofile-correction -DNO_NORETURN=1 + export CCACHE_DISABLE = t + V = 1 +endif +endif +endif + +# Every C object already waits for Git's forced GIT-CFLAGS target. Gate that +# target so "make strip install" finishes PGO before its normal prerequisites +# can start compiling with profile-use flags. +ifeq "$(OPENAI_PROFILE)" "BUILD" +GIT-CFLAGS: openai-profile +endif + +openai-profile: profile-clean openai-profile-clean + $(MAKE) OPENAI_PROFILE=GEN all + $(SHELL_PATH) $(OPENAI_PROFILE_TRAINING) + $(MAKE) OPENAI_PROFILE= openai-profile-merge + $(MAKE) OPENAI_PROFILE=USE all + +openai-profile-merge: +ifeq ("$(OPENAI_PROFILE_FORMAT)","LLVM") + $(OPENAI_LLVM_PROFDATA) merge \ + -output=$(OPENAI_PROFILE_DATA) \ + $(OPENAI_PROFILE_DIR)/*.profraw +endif + +clean: openai-profile-clean + +openai-profile-clean: + $(RM) $(OPENAI_PROFILE_DIR)/*.profraw $(OPENAI_PROFILE_DATA) + +.PHONY: openai-profile openai-profile-merge openai-profile-clean