Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 60 additions & 0 deletions .github/workflows/codex-pgo-training.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
#!/bin/sh

# Keep this workload short and biased toward the local Git operations Codex
# invokes frequently. The full Git test suite is too slow for each release
# target and would weight test-harness paths more heavily than status, diff,
# clone, fetch, and repository maintenance.

set -eu

git_bin="$PWD/bin-wrappers/git"
training_dir=$(mktemp -d "${TMPDIR:-/tmp}/codex-git-pgo.XXXXXX")
repo="$training_dir/repo"
clone="$training_dir/clone"

cleanup () {
rm -rf "$training_dir"
}
trap cleanup EXIT HUP INT TERM

mkdir -p "$training_dir/home"
export HOME="$training_dir/home"
export GIT_CONFIG_NOSYSTEM=1
export GIT_TERMINAL_PROMPT=0

"$git_bin" clone --quiet --no-local "$PWD" "$repo"
"$git_bin" -C "$repo" config user.name "Codex Git PGO"
"$git_bin" -C "$repo" config user.email "codex-git-pgo@openai.com"

i=0
while test "$i" -lt 256
do
dir="$repo/training/$((i % 16))"
mkdir -p "$dir"
printf '%s\n' "$i" >"$dir/file-$i"
i=$((i + 1))
done

"$git_bin" -C "$repo" status --porcelain=v2 --branch >/dev/null
"$git_bin" -C "$repo" status --porcelain=v2 --branch --untracked-files=all >/dev/null
"$git_bin" -C "$repo" ls-files --others --exclude-standard >/dev/null
"$git_bin" -C "$repo" add training
"$git_bin" -C "$repo" diff --cached --stat >/dev/null
"$git_bin" -C "$repo" commit --quiet -m "add training files"

printf 'changed\n' >>"$repo/training/0/file-0"
rm "$repo/training/1/file-1"
mkdir -p "$repo/untracked"
printf 'new\n' >"$repo/untracked/file"

"$git_bin" -C "$repo" status --porcelain=v2 --branch >/dev/null
"$git_bin" -C "$repo" diff --stat >/dev/null
"$git_bin" -C "$repo" diff --name-status >/dev/null
"$git_bin" -C "$repo" ls-files --stage >/dev/null
"$git_bin" -C "$repo" log --oneline --decorate -20 >/dev/null
"$git_bin" -C "$repo" rev-list --objects --all >/dev/null
"$git_bin" -C "$repo" for-each-ref --format='%(refname) %(objectname)' >/dev/null
"$git_bin" -C "$repo" repack -ad
"$git_bin" clone --quiet --no-local "$repo" "$clone"
"$git_bin" -C "$clone" status --porcelain=v2 --branch >/dev/null
"$git_bin" -C "$clone" fetch --quiet "$repo"
65 changes: 48 additions & 17 deletions .github/workflows/codex-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ jobs:
name: ${{ matrix.name }}
needs: version
runs-on: ${{ matrix.os }}
timeout-minutes: 30
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
Expand All @@ -69,6 +69,9 @@ jobs:
binary: /tmp/build/git/bin/git
file_pattern: Mach-O 64-bit executable arm64
has_gcm: false
lto: thin
profile_format: LLVM
llvm_profdata: xcrun llvm-profdata
max_tar_bytes: 67108864
- name: macOS x64
os: macos-15-intel
Expand All @@ -78,15 +81,21 @@ jobs:
binary: /tmp/build/git/bin/git
file_pattern: Mach-O 64-bit executable x86_64
has_gcm: false
lto: thin
profile_format: LLVM
llvm_profdata: xcrun llvm-profdata
max_tar_bytes: 67108864
# Keep arm64 builds native so release smoke tests can execute them.
- name: Linux arm64
os: ubuntu-22.04
os: ubuntu-22.04-arm
target_platform: ubuntu
asset_platform: ubuntu
arch: arm64
binary: /tmp/build/git/bin/git
file_pattern: ELF 64-bit.*ARM aarch64
has_gcm: false
lto: auto
profile_format: GCC
max_tar_bytes: 67108864
- name: Linux x64
os: ubuntu-22.04
Expand All @@ -96,6 +105,8 @@ jobs:
binary: /tmp/build/git/bin/git
file_pattern: ELF 64-bit.*x86-64
has_gcm: false
lto: auto
profile_format: GCC
max_tar_bytes: 67108864
- name: Windows arm64
os: windows-11-arm
Expand All @@ -105,6 +116,9 @@ jobs:
binary: /tmp/build/git/clangarm64/bin/git.exe
file_pattern: PE32\+.*ARM64
has_gcm: true
lto: thin
profile_format: LLVM
llvm_profdata: llvm-profdata
max_tar_bytes: 134217728
sdk_arch: aarch64
sdk_flavor: full
Expand All @@ -121,6 +135,8 @@ jobs:
binary: /tmp/build/git/mingw64/bin/git.exe
file_pattern: PE32\+.*x86-64
has_gcm: true
lto: auto
profile_format: GCC
max_tar_bytes: 134217728
sdk_arch: x86_64
sdk_flavor: full
Expand Down Expand Up @@ -161,6 +177,11 @@ jobs:
-c 'user.email=41898282+github-actions[bot]@users.noreply.github.com' \
tag -a "$VERSION" -m "$VERSION"

- name: Install OpenAI build configuration
shell: bash
working-directory: dugite-native/git
run: cp config.mak.openai config.mak

# Match Dugite Native's compatibility choice for its macOS x64 build.
- name: Select Xcode 16.4
if: matrix.target_platform == 'macOS' && matrix.arch == 'x64'
Expand Down Expand Up @@ -195,19 +216,13 @@ jobs:
- name: Install Linux arm64 build dependencies
if: matrix.target_platform == 'ubuntu' && matrix.arch == 'arm64'
run: |
sudo sed -i "s/^deb/deb [arch=amd64,i386]/g" /etc/apt/sources.list
release="$(lsb_release -s -c)"
echo "deb [arch=arm64,armhf] http://azure.ports.ubuntu.com/ ${release} main universe multiverse restricted" | sudo tee -a /etc/apt/sources.list
echo "deb [arch=arm64,armhf] http://azure.ports.ubuntu.com/ ${release}-updates main universe multiverse restricted" | sudo tee -a /etc/apt/sources.list
sudo dpkg --add-architecture arm64
sudo apt-get update
sudo apt-get install -y \
binutils-aarch64-linux-gnu \
gcc-aarch64-linux-gnu \
libcurl4-gnutls-dev:arm64 \
libexpat1-dev:arm64 \
libssl-dev:arm64 \
zlib1g-dev:arm64
libcurl4-gnutls-dev \
libexpat1-dev \
libssl-dev \
zlib1g-dev

# Dugite Native currently pins MinGit 2.53. Keep its build script and
# dependency schema, but match the runtime to the Git series we compile.
Expand Down Expand Up @@ -255,13 +270,18 @@ jobs:
dependencies.json >"$updated"
mv "$updated" dependencies.json

# Dugite cross-compiles several targets. Keep Git's optional Rust
# library disabled until its Makefile can direct Cargo at those targets.
# Build an instrumented Git, run a representative local workload, then
# rebuild with its profile. Keep Git's optional Rust library disabled
# until its Makefile can direct Cargo at these targets.
- name: Build the Dugite Native distribution
shell: bash
working-directory: dugite-native
env:
NO_RUST: 1
OPENAI_LLVM_PROFDATA: ${{ matrix.llvm_profdata }}
OPENAI_LTO: ${{ matrix.lto }}
OPENAI_PROFILE: BUILD
OPENAI_PROFILE_FORMAT: ${{ matrix.profile_format }}
TARGET_PLATFORM: ${{ matrix.target_platform }}
TARGET_ARCH: ${{ matrix.arch }}
run: |
Expand All @@ -287,6 +307,9 @@ jobs:
working-directory: dugite-native/git
env:
MINGW_DIR: ${{ matrix.mingw_dir }}
OPENAI_LLVM_PROFDATA: ${{ matrix.llvm_profdata }}
OPENAI_LTO: ${{ matrix.lto }}
OPENAI_PROFILE_FORMAT: ${{ matrix.profile_format }}
run: |
set -euo pipefail

Expand All @@ -301,8 +324,8 @@ jobs:
SKIP_DASHED_BUILT_INS=YesPlease
)
jobs="${NUMBER_OF_PROCESSORS:-2}"
make -j"$jobs" "${make_args[@]}" all
make "${make_args[@]}" DESTDIR=/tmp/build/git strip install
make -j"$jobs" "${make_args[@]}" OPENAI_PROFILE=BUILD all
make "${make_args[@]}" OPENAI_PROFILE=USE DESTDIR=/tmp/build/git strip install

- name: Verify distribution layout and provenance
shell: bash
Expand All @@ -312,6 +335,8 @@ jobs:
GIT_BINARY: ${{ matrix.binary }}
FILE_PATTERN: ${{ matrix.file_pattern }}
HAS_GCM: ${{ matrix.has_gcm }}
LTO: ${{ matrix.lto }}
PROFILE_FORMAT: ${{ matrix.profile_format }}
run: |
set -euo pipefail
if test "$TARGET_PLATFORM" = win32
Expand Down Expand Up @@ -339,9 +364,15 @@ jobs:
file "$GIT_BINARY" | tee /tmp/git-file-type
grep -E "$FILE_PATTERN" /tmp/git-file-type
strings "$GIT_BINARY" | grep -F "$GITHUB_SHA"
grep -F -- "-flto=$LTO" dugite-native/git/GIT-CFLAGS
if test "$PROFILE_FORMAT" = LLVM
then
grep -F -- "-fprofile-instr-use=" dugite-native/git/GIT-CFLAGS
else
grep -F -- "-fprofile-use=" dugite-native/git/GIT-CFLAGS
fi

- name: Smoke-test the native distribution
if: matrix.arch == 'x64' || matrix.target_platform == 'win32'
shell: bash
env:
TARGET_PLATFORM: ${{ matrix.target_platform }}
Expand Down
74 changes: 74 additions & 0 deletions config.mak.openai
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
# OpenAI release build settings.
#
# The Codex release workflow copies this file to config.mak before
# building. Keep release-only optimizations here instead of in the
# upstream Makefile.

OPENAI_PROFILE_DIR := $(CURDIR)
OPENAI_PROFILE_RAW := $(OPENAI_PROFILE_DIR)/default_%m_%p.profraw
OPENAI_PROFILE_DATA := $(OPENAI_PROFILE_DIR)/default.profdata
OPENAI_PROFILE_TRAINING ?= .github/workflows/codex-pgo-training.sh
OPENAI_LLVM_PROFDATA ?= llvm-profdata

ifdef OPENAI_LTO
CFLAGS_APPEND += -flto=$(OPENAI_LTO)
LDFLAGS_APPEND += -flto=$(OPENAI_LTO)
endif

ifeq ("$(OPENAI_PROFILE_FORMAT)","LLVM")
ifeq ("$(OPENAI_PROFILE)","GEN")
BASIC_CFLAGS += -fprofile-instr-generate=$(OPENAI_PROFILE_RAW)
BASIC_CFLAGS += -DNO_NORETURN=1
export CCACHE_DISABLE = t
V = 1
else
ifneq ("$(OPENAI_PROFILE)","")
BASIC_CFLAGS += -fprofile-instr-use=$(OPENAI_PROFILE_DATA)
BASIC_CFLAGS += -Wno-profile-instr-unprofiled -DNO_NORETURN=1
export CCACHE_DISABLE = t
V = 1
endif
endif
else
ifeq ("$(OPENAI_PROFILE)","GEN")
BASIC_CFLAGS += -fprofile-generate=$(OPENAI_PROFILE_DIR)
BASIC_CFLAGS += -DNO_NORETURN=1
EXTLIBS += -lgcov
export CCACHE_DISABLE = t
V = 1
else
ifneq ("$(OPENAI_PROFILE)","")
BASIC_CFLAGS += -fprofile-use=$(OPENAI_PROFILE_DIR)
BASIC_CFLAGS += -fprofile-correction -DNO_NORETURN=1
export CCACHE_DISABLE = t
V = 1
endif
endif
endif

# Every C object already waits for Git's forced GIT-CFLAGS target. Gate that
# target so "make strip install" finishes PGO before its normal prerequisites
# can start compiling with profile-use flags.
ifeq "$(OPENAI_PROFILE)" "BUILD"
GIT-CFLAGS: openai-profile
endif

openai-profile: profile-clean openai-profile-clean
$(MAKE) OPENAI_PROFILE=GEN all
$(SHELL_PATH) $(OPENAI_PROFILE_TRAINING)
$(MAKE) OPENAI_PROFILE= openai-profile-merge
$(MAKE) OPENAI_PROFILE=USE all

openai-profile-merge:
ifeq ("$(OPENAI_PROFILE_FORMAT)","LLVM")
$(OPENAI_LLVM_PROFDATA) merge \
-output=$(OPENAI_PROFILE_DATA) \
$(OPENAI_PROFILE_DIR)/*.profraw
endif

clean: openai-profile-clean

openai-profile-clean:
$(RM) $(OPENAI_PROFILE_DIR)/*.profraw $(OPENAI_PROFILE_DATA)

.PHONY: openai-profile openai-profile-merge openai-profile-clean
Loading