diff --git a/.github/workflows/codex-pgo-training.sh b/.github/workflows/codex-pgo-training.sh new file mode 100644 index 00000000000000..070d8b54cae486 --- /dev/null +++ b/.github/workflows/codex-pgo-training.sh @@ -0,0 +1,60 @@ +#!/bin/sh + +# Keep this workload short and biased toward the local Git operations Codex +# invokes frequently. The full Git test suite is too slow for each release +# target and would weight test-harness paths more heavily than status, diff, +# clone, fetch, and repository maintenance. + +set -eu + +git_bin="$PWD/bin-wrappers/git" +training_dir=$(mktemp -d "${TMPDIR:-/tmp}/codex-git-pgo.XXXXXX") +repo="$training_dir/repo" +clone="$training_dir/clone" + +cleanup () { + rm -rf "$training_dir" +} +trap cleanup EXIT HUP INT TERM + +mkdir -p "$training_dir/home" +export HOME="$training_dir/home" +export GIT_CONFIG_NOSYSTEM=1 +export GIT_TERMINAL_PROMPT=0 + +"$git_bin" clone --quiet --no-local "$PWD" "$repo" +"$git_bin" -C "$repo" config user.name "Codex Git PGO" +"$git_bin" -C "$repo" config user.email "codex-git-pgo@openai.com" + +i=0 +while test "$i" -lt 256 +do + dir="$repo/training/$((i % 16))" + mkdir -p "$dir" + printf '%s\n' "$i" >"$dir/file-$i" + i=$((i + 1)) +done + +"$git_bin" -C "$repo" status --porcelain=v2 --branch >/dev/null +"$git_bin" -C "$repo" status --porcelain=v2 --branch --untracked-files=all >/dev/null +"$git_bin" -C "$repo" ls-files --others --exclude-standard >/dev/null +"$git_bin" -C "$repo" add training +"$git_bin" -C "$repo" diff --cached --stat >/dev/null +"$git_bin" -C "$repo" commit --quiet -m "add training files" + +printf 'changed\n' >>"$repo/training/0/file-0" +rm "$repo/training/1/file-1" +mkdir -p "$repo/untracked" +printf 'new\n' >"$repo/untracked/file" + +"$git_bin" -C "$repo" status --porcelain=v2 --branch >/dev/null +"$git_bin" -C "$repo" diff --stat >/dev/null +"$git_bin" -C "$repo" diff --name-status >/dev/null +"$git_bin" -C "$repo" ls-files --stage >/dev/null +"$git_bin" -C "$repo" log --oneline --decorate -20 >/dev/null +"$git_bin" -C "$repo" rev-list --objects --all >/dev/null +"$git_bin" -C "$repo" for-each-ref --format='%(refname) %(objectname)' >/dev/null +"$git_bin" -C "$repo" repack -ad +"$git_bin" clone --quiet --no-local "$repo" "$clone" +"$git_bin" -C "$clone" status --porcelain=v2 --branch >/dev/null +"$git_bin" -C "$clone" fetch --quiet "$repo" diff --git a/.github/workflows/codex-release.yml b/.github/workflows/codex-release.yml index 98ffccc6e23e19..bf0ea496175a04 100644 --- a/.github/workflows/codex-release.yml +++ b/.github/workflows/codex-release.yml @@ -56,7 +56,7 @@ jobs: name: ${{ matrix.name }} needs: version runs-on: ${{ matrix.os }} - timeout-minutes: 30 + timeout-minutes: 60 strategy: fail-fast: false matrix: @@ -69,6 +69,9 @@ jobs: binary: /tmp/build/git/bin/git file_pattern: Mach-O 64-bit executable arm64 has_gcm: false + lto: thin + profile_format: LLVM + llvm_profdata: xcrun llvm-profdata max_tar_bytes: 67108864 - name: macOS x64 os: macos-15-intel @@ -78,15 +81,21 @@ jobs: binary: /tmp/build/git/bin/git file_pattern: Mach-O 64-bit executable x86_64 has_gcm: false + lto: thin + profile_format: LLVM + llvm_profdata: xcrun llvm-profdata max_tar_bytes: 67108864 + # Keep arm64 builds native so release smoke tests can execute them. - name: Linux arm64 - os: ubuntu-22.04 + os: ubuntu-22.04-arm target_platform: ubuntu asset_platform: ubuntu arch: arm64 binary: /tmp/build/git/bin/git file_pattern: ELF 64-bit.*ARM aarch64 has_gcm: false + lto: auto + profile_format: GCC max_tar_bytes: 67108864 - name: Linux x64 os: ubuntu-22.04 @@ -96,6 +105,8 @@ jobs: binary: /tmp/build/git/bin/git file_pattern: ELF 64-bit.*x86-64 has_gcm: false + lto: auto + profile_format: GCC max_tar_bytes: 67108864 - name: Windows arm64 os: windows-11-arm @@ -105,6 +116,9 @@ jobs: binary: /tmp/build/git/clangarm64/bin/git.exe file_pattern: PE32\+.*ARM64 has_gcm: true + lto: thin + profile_format: LLVM + llvm_profdata: llvm-profdata max_tar_bytes: 134217728 sdk_arch: aarch64 sdk_flavor: full @@ -121,6 +135,8 @@ jobs: binary: /tmp/build/git/mingw64/bin/git.exe file_pattern: PE32\+.*x86-64 has_gcm: true + lto: auto + profile_format: GCC max_tar_bytes: 134217728 sdk_arch: x86_64 sdk_flavor: full @@ -161,6 +177,11 @@ jobs: -c 'user.email=41898282+github-actions[bot]@users.noreply.github.com' \ tag -a "$VERSION" -m "$VERSION" + - name: Install OpenAI build configuration + shell: bash + working-directory: dugite-native/git + run: cp config.mak.openai config.mak + # Match Dugite Native's compatibility choice for its macOS x64 build. - name: Select Xcode 16.4 if: matrix.target_platform == 'macOS' && matrix.arch == 'x64' @@ -195,19 +216,13 @@ jobs: - name: Install Linux arm64 build dependencies if: matrix.target_platform == 'ubuntu' && matrix.arch == 'arm64' run: | - sudo sed -i "s/^deb/deb [arch=amd64,i386]/g" /etc/apt/sources.list - release="$(lsb_release -s -c)" - echo "deb [arch=arm64,armhf] http://azure.ports.ubuntu.com/ ${release} main universe multiverse restricted" | sudo tee -a /etc/apt/sources.list - echo "deb [arch=arm64,armhf] http://azure.ports.ubuntu.com/ ${release}-updates main universe multiverse restricted" | sudo tee -a /etc/apt/sources.list - sudo dpkg --add-architecture arm64 - sudo apt-get update sudo apt-get install -y \ binutils-aarch64-linux-gnu \ gcc-aarch64-linux-gnu \ - libcurl4-gnutls-dev:arm64 \ - libexpat1-dev:arm64 \ - libssl-dev:arm64 \ - zlib1g-dev:arm64 + libcurl4-gnutls-dev \ + libexpat1-dev \ + libssl-dev \ + zlib1g-dev # Dugite Native currently pins MinGit 2.53. Keep its build script and # dependency schema, but match the runtime to the Git series we compile. @@ -255,13 +270,18 @@ jobs: dependencies.json >"$updated" mv "$updated" dependencies.json - # Dugite cross-compiles several targets. Keep Git's optional Rust - # library disabled until its Makefile can direct Cargo at those targets. + # Build an instrumented Git, run a representative local workload, then + # rebuild with its profile. Keep Git's optional Rust library disabled + # until its Makefile can direct Cargo at these targets. - name: Build the Dugite Native distribution shell: bash working-directory: dugite-native env: NO_RUST: 1 + OPENAI_LLVM_PROFDATA: ${{ matrix.llvm_profdata }} + OPENAI_LTO: ${{ matrix.lto }} + OPENAI_PROFILE: BUILD + OPENAI_PROFILE_FORMAT: ${{ matrix.profile_format }} TARGET_PLATFORM: ${{ matrix.target_platform }} TARGET_ARCH: ${{ matrix.arch }} run: | @@ -287,6 +307,9 @@ jobs: working-directory: dugite-native/git env: MINGW_DIR: ${{ matrix.mingw_dir }} + OPENAI_LLVM_PROFDATA: ${{ matrix.llvm_profdata }} + OPENAI_LTO: ${{ matrix.lto }} + OPENAI_PROFILE_FORMAT: ${{ matrix.profile_format }} run: | set -euo pipefail @@ -301,8 +324,8 @@ jobs: SKIP_DASHED_BUILT_INS=YesPlease ) jobs="${NUMBER_OF_PROCESSORS:-2}" - make -j"$jobs" "${make_args[@]}" all - make "${make_args[@]}" DESTDIR=/tmp/build/git strip install + make -j"$jobs" "${make_args[@]}" OPENAI_PROFILE=BUILD all + make "${make_args[@]}" OPENAI_PROFILE=USE DESTDIR=/tmp/build/git strip install - name: Verify distribution layout and provenance shell: bash @@ -312,6 +335,8 @@ jobs: GIT_BINARY: ${{ matrix.binary }} FILE_PATTERN: ${{ matrix.file_pattern }} HAS_GCM: ${{ matrix.has_gcm }} + LTO: ${{ matrix.lto }} + PROFILE_FORMAT: ${{ matrix.profile_format }} run: | set -euo pipefail if test "$TARGET_PLATFORM" = win32 @@ -339,9 +364,15 @@ jobs: file "$GIT_BINARY" | tee /tmp/git-file-type grep -E "$FILE_PATTERN" /tmp/git-file-type strings "$GIT_BINARY" | grep -F "$GITHUB_SHA" + grep -F -- "-flto=$LTO" dugite-native/git/GIT-CFLAGS + if test "$PROFILE_FORMAT" = LLVM + then + grep -F -- "-fprofile-instr-use=" dugite-native/git/GIT-CFLAGS + else + grep -F -- "-fprofile-use=" dugite-native/git/GIT-CFLAGS + fi - name: Smoke-test the native distribution - if: matrix.arch == 'x64' || matrix.target_platform == 'win32' shell: bash env: TARGET_PLATFORM: ${{ matrix.target_platform }} diff --git a/config.mak.openai b/config.mak.openai new file mode 100644 index 00000000000000..0ae133c8bb2ecf --- /dev/null +++ b/config.mak.openai @@ -0,0 +1,74 @@ +# OpenAI release build settings. +# +# The Codex release workflow copies this file to config.mak before +# building. Keep release-only optimizations here instead of in the +# upstream Makefile. + +OPENAI_PROFILE_DIR := $(CURDIR) +OPENAI_PROFILE_RAW := $(OPENAI_PROFILE_DIR)/default_%m_%p.profraw +OPENAI_PROFILE_DATA := $(OPENAI_PROFILE_DIR)/default.profdata +OPENAI_PROFILE_TRAINING ?= .github/workflows/codex-pgo-training.sh +OPENAI_LLVM_PROFDATA ?= llvm-profdata + +ifdef OPENAI_LTO +CFLAGS_APPEND += -flto=$(OPENAI_LTO) +LDFLAGS_APPEND += -flto=$(OPENAI_LTO) +endif + +ifeq ("$(OPENAI_PROFILE_FORMAT)","LLVM") +ifeq ("$(OPENAI_PROFILE)","GEN") + BASIC_CFLAGS += -fprofile-instr-generate=$(OPENAI_PROFILE_RAW) + BASIC_CFLAGS += -DNO_NORETURN=1 + export CCACHE_DISABLE = t + V = 1 +else +ifneq ("$(OPENAI_PROFILE)","") + BASIC_CFLAGS += -fprofile-instr-use=$(OPENAI_PROFILE_DATA) + BASIC_CFLAGS += -Wno-profile-instr-unprofiled -DNO_NORETURN=1 + export CCACHE_DISABLE = t + V = 1 +endif +endif +else +ifeq ("$(OPENAI_PROFILE)","GEN") + BASIC_CFLAGS += -fprofile-generate=$(OPENAI_PROFILE_DIR) + BASIC_CFLAGS += -DNO_NORETURN=1 + EXTLIBS += -lgcov + export CCACHE_DISABLE = t + V = 1 +else +ifneq ("$(OPENAI_PROFILE)","") + BASIC_CFLAGS += -fprofile-use=$(OPENAI_PROFILE_DIR) + BASIC_CFLAGS += -fprofile-correction -DNO_NORETURN=1 + export CCACHE_DISABLE = t + V = 1 +endif +endif +endif + +# Every C object already waits for Git's forced GIT-CFLAGS target. Gate that +# target so "make strip install" finishes PGO before its normal prerequisites +# can start compiling with profile-use flags. +ifeq "$(OPENAI_PROFILE)" "BUILD" +GIT-CFLAGS: openai-profile +endif + +openai-profile: profile-clean openai-profile-clean + $(MAKE) OPENAI_PROFILE=GEN all + $(SHELL_PATH) $(OPENAI_PROFILE_TRAINING) + $(MAKE) OPENAI_PROFILE= openai-profile-merge + $(MAKE) OPENAI_PROFILE=USE all + +openai-profile-merge: +ifeq ("$(OPENAI_PROFILE_FORMAT)","LLVM") + $(OPENAI_LLVM_PROFDATA) merge \ + -output=$(OPENAI_PROFILE_DATA) \ + $(OPENAI_PROFILE_DIR)/*.profraw +endif + +clean: openai-profile-clean + +openai-profile-clean: + $(RM) $(OPENAI_PROFILE_DIR)/*.profraw $(OPENAI_PROFILE_DATA) + +.PHONY: openai-profile openai-profile-merge openai-profile-clean