From fbecab78a817e97ae9262a65532dad3ac49fdbc7 Mon Sep 17 00:00:00 2001 From: Adam Perry Date: Sat, 20 Jun 2026 02:47:11 +0000 Subject: [PATCH 01/10] build: remove MSVC from Windows Bazel build actions --- .bazelrc | 4 +- .github/actions/setup-bazel-ci/action.yml | 59 +------ .../scripts/compute-bazel-windows-path.ps1 | 134 +++++--------- .github/scripts/run-bazel-ci.sh | 112 ++++-------- .../scripts/test_run_bazel_with_buildbuddy.py | 164 +++++++++++------- .github/workflows/bazel.yml | 20 +-- BUILD.bazel | 4 +- MODULE.bazel | 18 -- bazel/rules/testing/wine/wine.bzl | 3 + defs.bzl | 9 +- patches/BUILD.bazel | 2 - ...ws-lc-sys_windows_msvc_prebuilt_nasm.patch | 18 +- patches/bzip2_windows_stack_args.patch | 23 --- ...t_windows_exec_msvc_build_script_env.patch | 12 +- ...es_rust_windows_gnullvm_build_script.patch | 25 +-- patches/xz_windows_stack_args.patch | 14 -- scripts/list-bazel-clippy-targets.sh | 15 +- tools/windows-toolchain/BUILD.bazel | 12 ++ .../windows-toolchain/stack_protector_probe.c | 11 ++ 19 files changed, 240 insertions(+), 419 deletions(-) delete mode 100644 patches/bzip2_windows_stack_args.patch delete mode 100644 patches/xz_windows_stack_args.patch create mode 100644 tools/windows-toolchain/BUILD.bazel create mode 100644 tools/windows-toolchain/stack_protector_probe.c diff --git a/.bazelrc b/.bazelrc index 8814a69ca980..79f497b95317 100644 --- a/.bazelrc +++ b/.bazelrc @@ -207,8 +207,8 @@ common:ci-windows-cross --test_env=RUST_TEST_THREADS=1 # the hidden dynamic-tool callback test, which currently times out on Windows. common:ci-windows-cross --test_env=CODEX_BAZEL_TEST_SKIP_FILTERS=powershell,suite::code_mode::code_mode_can_call_hidden_dynamic_tools common:ci-windows-cross --platforms=//:windows_x86_64_gnullvm -common:ci-windows-cross --extra_execution_platforms=//:rbe,//:windows_x86_64_msvc -common:ci-windows-cross --extra_toolchains=//:windows_gnullvm_tests_on_msvc_host_toolchain +common:ci-windows-cross --extra_execution_platforms=//:rbe,//:windows_x86_64_gnullvm +common:ci-windows-cross --extra_toolchains=//:windows_gnullvm_tests_on_gnullvm_host_toolchain # Linux-only V8 CI config. common:ci-v8 --config=ci diff --git a/.github/actions/setup-bazel-ci/action.yml b/.github/actions/setup-bazel-ci/action.yml index bb757aab91de..3357e8fb2829 100644 --- a/.github/actions/setup-bazel-ci/action.yml +++ b/.github/actions/setup-bazel-ci/action.yml @@ -59,64 +59,7 @@ runs: "BAZEL_OUTPUT_USER_ROOT=$bazelOutputUserRoot" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append "BAZEL_REPO_CONTENTS_CACHE=$repoContentsCache" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - - name: Expose MSVC SDK environment (Windows) - if: runner.os == 'Windows' - shell: pwsh - run: | - # Bazel exec-side Rust build scripts do not reliably inherit the MSVC developer - # shell on GitHub-hosted Windows runners, so discover the latest VS install and - # ask `VsDevCmd.bat` to materialize the x64/x64 compiler + SDK environment. - $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" - if (-not (Test-Path $vswhere)) { - throw "vswhere.exe not found" - } - - $installPath = & $vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath 2>$null - if (-not $installPath) { - throw "Could not locate a Visual Studio installation with VC tools" - } - - $vsDevCmd = Join-Path $installPath 'Common7\Tools\VsDevCmd.bat' - if (-not (Test-Path $vsDevCmd)) { - throw "VsDevCmd.bat not found at $vsDevCmd" - } - - # Keep the export surface explicit: these are the paths and SDK roots that the - # MSVC toolchain probes need later when Bazel runs Windows exec-platform build - # scripts such as `aws-lc-sys`. - $varsToExport = @( - 'INCLUDE', - 'LIB', - 'LIBPATH', - 'PATH', - 'UCRTVersion', - 'UniversalCRTSdkDir', - 'VCINSTALLDIR', - 'VCToolsInstallDir', - 'WindowsLibPath', - 'WindowsSdkBinPath', - 'WindowsSdkDir', - 'WindowsSDKLibVersion', - 'WindowsSDKVersion' - ) - - # `VsDevCmd.bat` is a batch file, so invoke it under `cmd.exe`, suppress its - # banner, then dump the resulting environment with `set`. Re-export only the - # approved keys into `GITHUB_ENV` so later steps inherit the same MSVC context. - $envLines = & cmd.exe /c ('"{0}" -no_logo -arch=x64 -host_arch=x64 >nul && set' -f $vsDevCmd) - foreach ($line in $envLines) { - if ($line -notmatch '^(.*?)=(.*)$') { - continue - } - - $name = $matches[1] - $value = $matches[2] - if ($varsToExport -contains $name) { - "$name=$value" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - } - } - - - name: Compute cache-stable Windows Bazel PATH + - name: Configure frozen Windows Bazel paths if: runner.os == 'Windows' shell: pwsh run: ./.github/scripts/compute-bazel-windows-path.ps1 diff --git a/.github/scripts/compute-bazel-windows-path.ps1 b/.github/scripts/compute-bazel-windows-path.ps1 index 81fd668c8b53..347d16bfef4c 100644 --- a/.github/scripts/compute-bazel-windows-path.ps1 +++ b/.github/scripts/compute-bazel-windows-path.ps1 @@ -1,113 +1,59 @@ <# -BuildBuddy cache keys include the action and test environment, so Bazel should -not inherit the full hosted-runner PATH on Windows. That PATH includes volatile -tool entries, such as Maven, that can change independently of this repo and -cause avoidable cache misses. - -This script derives a smaller, cache-stable PATH that keeps the Windows -toolchain entries Bazel-backed CI tasks need: MSVC and Windows SDK paths, -MinGW runtime DLL paths for gnullvm-built tests, Git, PowerShell, Node, Python, -DotSlash, and the standard Windows system directories. -`setup-bazel-ci` runs this after exporting the MSVC environment, and the script -publishes the result via `GITHUB_ENV` as `CODEX_BAZEL_WINDOWS_PATH` so later -steps can pass that explicit PATH to Bazel. +Bazel build actions must not inherit the hosted-runner PATH. Keep their +execution substrate fixed to Windows and the Git-for-Windows shell utilities +that Bazel genrules already use. Test actions get a separate fixed path with +the product runtimes exercised by Windows tests (Git, PowerShell, and +DotSlash). Compiler, SDK, MinGW, hosted Python, and hosted Node directories are +intentionally absent from both values. #> -$stablePathEntries = New-Object System.Collections.Generic.List[string] -$seenEntries = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) -$windowsAppsPath = if ([string]::IsNullOrWhiteSpace($env:LOCALAPPDATA)) { - $null -} else { - "$($env:LOCALAPPDATA)\Microsoft\WindowsApps" -} $windowsDir = if ($env:WINDIR) { $env:WINDIR } elseif ($env:SystemRoot) { $env:SystemRoot } else { - $null + throw 'WINDIR or SystemRoot must be set.' } -function Add-StablePathEntry { - param([string]$PathEntry) - - if ([string]::IsNullOrWhiteSpace($PathEntry)) { - return - } - - if ($seenEntries.Add($PathEntry)) { - [void]$stablePathEntries.Add($PathEntry) - } +if ([string]::IsNullOrWhiteSpace($env:ProgramFiles)) { + throw 'ProgramFiles must be set.' } - -foreach ($pathEntry in ($env:PATH -split ';')) { - if ([string]::IsNullOrWhiteSpace($pathEntry)) { - continue - } - - if ( - $pathEntry -like '*Microsoft Visual Studio*' -or - $pathEntry -like '*Windows Kits*' -or - $pathEntry -like '*Microsoft SDKs*' -or - $pathEntry -eq 'C:\mingw64\bin' -or - $pathEntry -like 'C:\msys64\*\bin' -or - $pathEntry -like 'C:\Program Files\Git\*' -or - $pathEntry -like 'C:\Program Files\PowerShell\*' -or - $pathEntry -like 'C:\hostedtoolcache\windows\node\*' -or - $pathEntry -like 'C:\hostedtoolcache\windows\Python\*' -or - $pathEntry -eq 'D:\a\_temp\install-dotslash\bin' -or - ($windowsDir -and ($pathEntry -eq $windowsDir -or $pathEntry -like "${windowsDir}\*")) - ) { - Add-StablePathEntry $pathEntry - } -} - -$gitCommand = Get-Command git -ErrorAction SilentlyContinue -if ($gitCommand) { - Add-StablePathEntry (Split-Path $gitCommand.Source -Parent) -} - -$nodeCommand = Get-Command node -ErrorAction SilentlyContinue -if ($nodeCommand) { - Add-StablePathEntry (Split-Path $nodeCommand.Source -Parent) +if ([string]::IsNullOrWhiteSpace($env:LOCALAPPDATA)) { + throw 'LOCALAPPDATA must be set.' } - -$python3Command = Get-Command python3 -ErrorAction SilentlyContinue -if ($python3Command) { - Add-StablePathEntry (Split-Path $python3Command.Source -Parent) -} - -$pythonCommand = Get-Command python -ErrorAction SilentlyContinue -if ($pythonCommand) { - Add-StablePathEntry (Split-Path $pythonCommand.Source -Parent) -} - -$pwshCommand = Get-Command pwsh -ErrorAction SilentlyContinue -if ($pwshCommand) { - Add-StablePathEntry (Split-Path $pwshCommand.Source -Parent) +if ([string]::IsNullOrWhiteSpace($env:GITHUB_ENV)) { + throw 'GITHUB_ENV must be set.' } -foreach ($mingwPath in @('C:\mingw64\bin', 'C:\msys64\mingw64\bin', 'C:\msys64\ucrt64\bin')) { - if (Test-Path $mingwPath) { - Add-StablePathEntry $mingwPath +$gitRoot = Join-Path $env:ProgramFiles 'Git' +$executionPathEntries = @( + (Join-Path $gitRoot 'usr\bin'), + (Join-Path $windowsDir 'System32'), + $windowsDir +) +$testPathEntries = @( + (Join-Path $env:ProgramFiles 'PowerShell\7'), + (Join-Path $gitRoot 'bin'), + (Join-Path $gitRoot 'usr\bin'), + (Join-Path $env:LOCALAPPDATA 'Microsoft\WindowsApps'), + (Join-Path $windowsDir 'System32'), + $windowsDir +) + +$requiredPathEntries = ($executionPathEntries + $testPathEntries) | Select-Object -Unique +foreach ($pathEntry in $requiredPathEntries) { + if (-not (Test-Path $pathEntry)) { + throw "Required Windows Bazel substrate path does not exist: $pathEntry" } } -if ($windowsAppsPath) { - Add-StablePathEntry $windowsAppsPath -} +$executionPath = $executionPathEntries -join ';' +$testPath = $testPathEntries -join ';' -if ($stablePathEntries.Count -eq 0) { - throw 'Failed to derive cache-stable Windows PATH.' -} +Write-Host 'Frozen CODEX_BAZEL_WINDOWS_EXECUTION_PATH entries:' +$executionPathEntries | ForEach-Object { Write-Host " $_" } +Write-Host 'Frozen CODEX_BAZEL_WINDOWS_TEST_PATH entries:' +$testPathEntries | ForEach-Object { Write-Host " $_" } -if ([string]::IsNullOrWhiteSpace($env:GITHUB_ENV)) { - throw 'GITHUB_ENV must be set.' -} - -$stablePath = $stablePathEntries -join ';' -Write-Host 'Derived CODEX_BAZEL_WINDOWS_PATH entries:' -foreach ($pathEntry in $stablePathEntries) { - Write-Host " $pathEntry" -} -"CODEX_BAZEL_WINDOWS_PATH=$stablePath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append +"CODEX_BAZEL_WINDOWS_EXECUTION_PATH=$executionPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append +"CODEX_BAZEL_WINDOWS_TEST_PATH=$testPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append diff --git a/.github/scripts/run-bazel-ci.sh b/.github/scripts/run-bazel-ci.sh index 58114a32d6ce..79aa630172a1 100755 --- a/.github/scripts/run-bazel-ci.sh +++ b/.github/scripts/run-bazel-ci.sh @@ -5,7 +5,6 @@ set -euo pipefail print_failed_bazel_test_logs=0 print_failed_bazel_action_summary=0 remote_download_toplevel=0 -windows_msvc_host_platform=0 windows_cross_compile=0 windows_hybrid_execution=0 @@ -23,10 +22,6 @@ while [[ $# -gt 0 ]]; do remote_download_toplevel=1 shift ;; - --windows-msvc-host-platform) - windows_msvc_host_platform=1 - shift - ;; --windows-cross-compile) windows_cross_compile=1 shift @@ -47,7 +42,7 @@ while [[ $# -gt 0 ]]; do done if [[ $# -eq 0 ]]; then - echo "Usage: $0 [--print-failed-test-logs] [--print-failed-action-summary] [--remote-download-toplevel] [--windows-msvc-host-platform] [--windows-cross-compile] [--windows-hybrid-execution] -- -- " >&2 + echo "Usage: $0 [--print-failed-test-logs] [--print-failed-action-summary] [--remote-download-toplevel] [--windows-cross-compile] [--windows-hybrid-execution] -- -- " >&2 exit 1 fi @@ -102,9 +97,7 @@ print_bazel_test_log_tails() { local -a bazel_info_args=(info) if [[ -n "${BUILDBUDDY_API_KEY:-}" ]]; then # `bazel info` needs the same CI config as the failed test invocation so - # platform-specific output roots match. On Windows, omitting `ci-windows` - # would point at `local_windows-fastbuild` even when the test ran with the - # MSVC host platform under `local_windows_msvc-fastbuild`. + # platform-specific output roots match. bazel_info_args+=("--config=${ci_config}") fi @@ -113,7 +106,7 @@ print_bazel_test_log_tails() { # mode can make `bazel info` fail, which would hide the real test log path. for arg in "${post_config_bazel_args[@]}"; do case "$arg" in - --host_platform=* | --repo_contents_cache=* | --repository_cache=*) + --host_platform=* | --platforms=* | --repo_contents_cache=* | --repository_cache=*) bazel_info_args+=("$arg") ;; esac @@ -267,31 +260,7 @@ if [[ ${#bazel_args[@]} -eq 0 || ${#bazel_targets[@]} -eq 0 ]]; then exit 1 fi -if [[ "${RUNNER_OS:-}" == "Windows" && $windows_cross_compile -eq 1 && -z "${BUILDBUDDY_API_KEY:-}" ]]; then - # Windows cross-compilation depends on authenticated RBE. Preserve the local - # Windows build shape when credentials are unavailable. - ci_config=ci-windows - windows_msvc_host_platform=1 -fi - post_config_bazel_args=() -if [[ "${RUNNER_OS:-}" == "Windows" && $windows_msvc_host_platform -eq 1 ]]; then - has_host_platform_override=0 - for arg in "${bazel_args[@]}"; do - if [[ "$arg" == --host_platform=* ]]; then - has_host_platform_override=1 - break - fi - done - - if [[ $has_host_platform_override -eq 0 ]]; then - # Use the MSVC Windows platform for jobs that need helper binaries like - # Rust test wrappers and V8 generators to resolve a compatible toolchain. - # Callers that need a different Windows target platform should pass an - # explicit `--platforms=...` flag. - post_config_bazel_args+=("--host_platform=//:local_windows_msvc") - fi -fi if [[ $remote_download_toplevel -eq 1 ]]; then # Override the CI config's remote_download_minimal setting when callers need @@ -308,16 +277,22 @@ if [[ "${RUNNER_OS:-}" == "Windows" && -n "${BUILDBUDDY_API_KEY:-}" && ( $window # `--enable_platform_specific_config` expands `common:windows` on Windows # hosts after ordinary rc configs, which can override `ci-windows-cross`'s # RBE host platform. Repeat it on the command line for cross builds. The - # Hybrid execution keeps its gnullvm host platform for local Rust actions. + # hybrid execution keeps its gnullvm host platform for local Rust actions. post_config_bazel_args+=(--host_platform=//:rbe) fi fi if [[ "${RUNNER_OS:-}" == "Windows" && $windows_cross_compile -eq 1 && -z "${BUILDBUDDY_API_KEY:-}" ]]; then # The Windows cross-compile config depends on authenticated remote - # execution. When credentials are unavailable, keep the local build shape - # and its lower concurrency cap. - post_config_bazel_args+=(--jobs=8) + # execution. When credentials are unavailable, spell out the equivalent + # local gnullvm platforms and keep the lower concurrency cap. + post_config_bazel_args+=( + --host_platform=//:local_windows + --platforms=//:windows_x86_64_gnullvm + --extra_execution_platforms=//:windows_x86_64_gnullvm + --extra_toolchains=//:windows_gnullvm_tests_on_gnullvm_host_toolchain + --jobs=8 + ) fi if [[ -n "${BAZEL_REPO_CONTENTS_CACHE:-}" ]]; then @@ -338,57 +313,28 @@ if [[ -n "${CODEX_BAZEL_EXECUTION_LOG_COMPACT_DIR:-}" ]]; then fi if [[ "${RUNNER_OS:-}" == "Windows" ]]; then - pass_windows_build_env=1 - if [[ -n "${BUILDBUDDY_API_KEY:-}" && ( $windows_cross_compile -eq 1 || $windows_hybrid_execution -eq 1 ) ]]; then - # Generic build actions execute on Linux RBE workers. Passing the Windows - # runner's compiler environment there leaks VS/SDK paths and makes genrules - # try to execute tools that do not exist on the worker. - pass_windows_build_env=0 - fi - - if [[ $pass_windows_build_env -eq 1 ]]; then - windows_action_env_vars=( - INCLUDE - LIB - LIBPATH - UCRTVersion - UniversalCRTSdkDir - VCINSTALLDIR - VCToolsInstallDir - WindowsLibPath - WindowsSdkBinPath - WindowsSdkDir - WindowsSDKLibVersion - WindowsSDKVersion - ) - - for env_var in "${windows_action_env_vars[@]}"; do - if [[ -n "${!env_var:-}" ]]; then - post_config_bazel_args+=("--action_env=${env_var}" "--host_action_env=${env_var}") - fi - done + if [[ -z "${CODEX_BAZEL_WINDOWS_EXECUTION_PATH:-}" ]]; then + echo "CODEX_BAZEL_WINDOWS_EXECUTION_PATH must be set for Windows Bazel CI." >&2 + exit 1 fi - - if [[ -z "${CODEX_BAZEL_WINDOWS_PATH:-}" ]]; then - echo "CODEX_BAZEL_WINDOWS_PATH must be set for Windows Bazel CI." >&2 + if [[ -z "${CODEX_BAZEL_WINDOWS_TEST_PATH:-}" ]]; then + echo "CODEX_BAZEL_WINDOWS_TEST_PATH must be set for Windows Bazel CI." >&2 exit 1 fi - if [[ $pass_windows_build_env -eq 1 ]]; then - post_config_bazel_args+=( - "--action_env=PATH=${CODEX_BAZEL_WINDOWS_PATH}" - "--host_action_env=PATH=${CODEX_BAZEL_WINDOWS_PATH}" - ) - else + windows_execution_path="${CODEX_BAZEL_WINDOWS_EXECUTION_PATH}" + if [[ -n "${BUILDBUDDY_API_KEY:-}" && ( $windows_cross_compile -eq 1 || $windows_hybrid_execution -eq 1 ) ]]; then # Remote build actions run on Linux RBE workers. Give their shell snippets - # a frozen Linux PATH while preserving CODEX_BAZEL_WINDOWS_PATH below only - # for local Windows test execution. - post_config_bazel_args+=( - "--action_env=PATH=/usr/bin:/bin" - "--host_action_env=PATH=/usr/bin:/bin" - ) + # a frozen Linux execution-substrate path. Windows Rust and build-script + # actions receive the same value, which intentionally cannot discover + # runner-installed Windows compilers or SDK tools. + windows_execution_path="/usr/bin:/bin" fi - post_config_bazel_args+=("--test_env=PATH=${CODEX_BAZEL_WINDOWS_PATH}") + post_config_bazel_args+=( + "--action_env=PATH=${windows_execution_path}" + "--host_action_env=PATH=${windows_execution_path}" + "--test_env=PATH=${CODEX_BAZEL_WINDOWS_TEST_PATH}" + ) fi bazel_console_log="$(mktemp)" diff --git a/.github/scripts/test_run_bazel_with_buildbuddy.py b/.github/scripts/test_run_bazel_with_buildbuddy.py index 299fbfc037cd..0c76835de25d 100644 --- a/.github/scripts/test_run_bazel_with_buildbuddy.py +++ b/.github/scripts/test_run_bazel_with_buildbuddy.py @@ -12,6 +12,66 @@ class RunBazelWithBuildBuddyTest(unittest.TestCase): + def run_bazel_ci( + self, + temp_dir: str, + env_overrides: dict[str, str], + *args: str, + ) -> list[str]: + fake_bazel_impl = Path(temp_dir) / "fake-bazel.py" + fake_bazel_impl.write_text( + "#!/usr/bin/env python3\n" + "import json\n" + "import sys\n" + "print(json.dumps(sys.argv[1:]))\n", + encoding="utf-8", + ) + if os.name == "nt": + fake_bazel = Path(temp_dir) / "fake-bazel.cmd" + fake_bazel.write_text( + f'@"{sys.executable}" "{fake_bazel_impl}" %*\n', + encoding="utf-8", + ) + else: + fake_bazel = fake_bazel_impl + fake_bazel.chmod(0o755) + + env = os.environ.copy() + for name in ( + "BUILDBUDDY_API_KEY", + "GITHUB_ACTIONS", + "GITHUB_EVENT_NAME", + "GITHUB_EVENT_PATH", + "GITHUB_REPOSITORY", + ): + env.pop(name, None) + env.update(env_overrides) + env["CODEX_BAZEL_BIN"] = str(fake_bazel) + + bash = "bash" + if os.name == "nt": + bash = str(Path(os.environ["ProgramFiles"]) / "Git" / "bin" / "bash.exe") + self.assertTrue(Path(bash).is_file(), bash) + + result = subprocess.run( + [ + bash, + str(Path(__file__).with_name("run-bazel-ci.sh")), + *args, + ], + env=env, + check=False, + capture_output=True, + text=True, + ) + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + return next( + json.loads(line) + for line in result.stdout.splitlines() + if line.startswith("[") + ) + def github_env( self, temp_dir: str, @@ -123,69 +183,21 @@ def test_windows_hybrid_execution_separates_build_and_test_environments( self, ) -> None: with TemporaryDirectory() as temp_dir: - fake_bazel_impl = Path(temp_dir) / "fake-bazel.py" - fake_bazel_impl.write_text( - "#!/usr/bin/env python3\n" - "import json\n" - "import sys\n" - "print(json.dumps(sys.argv[1:]))\n", - encoding="utf-8", - ) - if os.name == "nt": - fake_bazel = Path(temp_dir) / "fake-bazel.cmd" - fake_bazel.write_text( - f'@"{sys.executable}" "{fake_bazel_impl}" %*\n', - encoding="utf-8", - ) - else: - fake_bazel = fake_bazel_impl - fake_bazel.chmod(0o755) - - env = os.environ.copy() - for name in ( - "GITHUB_ACTIONS", - "GITHUB_EVENT_NAME", - "GITHUB_EVENT_PATH", - "GITHUB_REPOSITORY", - ): - env.pop(name, None) - env.update( + command = self.run_bazel_ci( + temp_dir, { "BUILDBUDDY_API_KEY": "token", - "CODEX_BAZEL_BIN": str(fake_bazel), - "CODEX_BAZEL_WINDOWS_PATH": r"C:\runtime\bin", + "CODEX_BAZEL_WINDOWS_EXECUTION_PATH": r"C:\substrate\bin", + "CODEX_BAZEL_WINDOWS_TEST_PATH": r"C:\runtime\bin", "INCLUDE": r"C:\Visual Studio\include", "RUNNER_OS": "Windows", - } - ) - - bash = "bash" - if os.name == "nt": - bash = str(Path(os.environ["ProgramFiles"]) / "Git" / "bin" / "bash.exe") - self.assertTrue(Path(bash).is_file(), bash) - - result = subprocess.run( - [ - bash, - str(Path(__file__).with_name("run-bazel-ci.sh")), - "--windows-hybrid-execution", - "--", - "build", - "--config=argument-comment-lint", - "--", - "//codex-rs/arg0:arg0", - ], - env=env, - check=False, - capture_output=True, - text=True, - ) - - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - command = next( - json.loads(line) - for line in result.stdout.splitlines() - if line.startswith("[") + }, + "--windows-hybrid-execution", + "--", + "build", + "--config=argument-comment-lint", + "--", + "//codex-rs/arg0:arg0", ) self.assertIn("--config=ci-windows-hybrid", command) self.assertIn("--shell_executable=/bin/bash", command) @@ -203,6 +215,40 @@ def test_windows_hybrid_execution_separates_build_and_test_environments( ) ) + def test_keyless_windows_cross_build_uses_gnullvm_substrate(self) -> None: + with TemporaryDirectory() as temp_dir: + command = self.run_bazel_ci( + temp_dir, + { + "CODEX_BAZEL_WINDOWS_EXECUTION_PATH": r"C:\substrate\bin", + "CODEX_BAZEL_WINDOWS_TEST_PATH": r"C:\runtime\bin", + "INCLUDE": r"C:\Visual Studio\include", + "RUNNER_OS": "Windows", + }, + "--windows-cross-compile", + "--", + "build", + "--", + "//codex-rs/arg0:arg0", + ) + + self.assertIn("--jobs=8", command) + self.assertIn("--host_platform=//:local_windows", command) + self.assertIn("--platforms=//:windows_x86_64_gnullvm", command) + self.assertIn( + "--extra_execution_platforms=//:windows_x86_64_gnullvm", command + ) + self.assertIn( + "--extra_toolchains=//:windows_gnullvm_tests_on_gnullvm_host_toolchain", + command, + ) + self.assertIn(r"--action_env=PATH=C:\substrate\bin", command) + self.assertIn(r"--host_action_env=PATH=C:\substrate\bin", command) + self.assertIn(r"--test_env=PATH=C:\runtime\bin", command) + self.assertFalse(any("msvc" in arg.lower() for arg in command)) + self.assertNotIn("--action_env=INCLUDE", command) + self.assertNotIn("--host_action_env=INCLUDE", command) + def test_query_remote_configuration_is_inserted_before_expression(self) -> None: expression = 'kind("rust_library rule", //codex-rs/...)' env = {"BUILDBUDDY_API_KEY": "fork-token"} diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 6b74e228c712..adedb6f22c9c 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -186,11 +186,12 @@ jobs: run: | set -euo pipefail - bazel_test_query='tests(//...) except tests(//third_party/v8:all) except attr(tags, "manual", tests(//...))' + bazel_test_query='tests(//...) except tests(//third_party/v8:all) except attr(tags, "manual", tests(//...)) except attr(tags, "windows-gnullvm-incompatible", tests(//...))' mapfile -t bazel_targets < <( ./.github/scripts/run-bazel-query-ci.sh --output=label -- "${bazel_test_query}" \ | LC_ALL=C sort ) + bazel_targets+=("//tools/windows-toolchain:stack-protector-probe") selected_targets=() for bazel_target in "${bazel_targets[@]}"; do @@ -213,7 +214,6 @@ jobs: bazel_test_args=( test - --skip_incompatible_explicit_targets --test_tag_filters=-argument-comment-lint --test_verbose_timeout_warnings --build_metadata=COMMIT_SHA=${GITHUB_SHA} @@ -386,17 +386,10 @@ jobs: if [[ "${RUNNER_OS}" == "Windows" ]]; then # Keep this aligned with the fast Windows Bazel test job: use # Linux RBE for clippy build actions while targeting Windows - # gnullvm. Fork/community PRs without the BuildBuddy secret fall - # back inside `run-bazel-ci.sh` to the previous local Windows MSVC - # host-platform shape. + # gnullvm. Fork/community PRs without the BuildBuddy secret use + # the same local Windows gnullvm host-platform shape. bazel_wrapper_args+=(--windows-cross-compile) bazel_target_list_args+=(--windows-cross-compile) - if [[ -z "${BUILDBUDDY_API_KEY:-}" ]]; then - # The fork fallback can see incompatible explicit Windows-cross - # internal test binaries in the generated target list. Preserve - # the old local-fallback behavior there. - bazel_clippy_args+=(--skip_incompatible_explicit_targets) - fi fi bazel_target_lines="$(./scripts/list-bazel-clippy-targets.sh "${bazel_target_list_args[@]}")" @@ -482,9 +475,8 @@ jobs: if [[ "${RUNNER_OS}" == "Windows" ]]; then # This is build-only signal, so use the same Linux-RBE # cross-compile path as the fast Windows test and clippy jobs. - # Fork/community PRs without the BuildBuddy secret fall back - # inside `run-bazel-ci.sh` to the previous local Windows MSVC - # host-platform shape. + # Fork/community PRs without the BuildBuddy secret use the same + # local Windows gnullvm host-platform shape. bazel_wrapper_args+=(--windows-cross-compile) fi diff --git a/BUILD.bazel b/BUILD.bazel index a82126e6f1ec..6e913825cb4a 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -49,11 +49,11 @@ platform( ) toolchain( - name = "windows_gnullvm_tests_on_msvc_host_toolchain", + name = "windows_gnullvm_tests_on_gnullvm_host_toolchain", exec_compatible_with = [ "@platforms//cpu:x86_64", "@platforms//os:windows", - "@rules_rs//rs/experimental/platforms/constraints:windows_msvc", + "@rules_rs//rs/experimental/platforms/constraints:windows_gnullvm", ], target_compatible_with = [ "@platforms//cpu:x86_64", diff --git a/MODULE.bazel b/MODULE.bazel index cd771adb1214..1e943122a2ee 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -106,10 +106,6 @@ single_version_override( ) rules_rust = use_extension("@rules_rs//rs/experimental:rules_rust.bzl", "rules_rust") - -# Build-script probe binaries inherit CFLAGS/CXXFLAGS from Bazel's C++ -# toolchain. On `windows-gnullvm`, llvm-mingw does not ship -# `libssp_nonshared`, so strip the forwarded stack-protector flags there. rules_rust.patch( patches = [ "//patches:rules_rust_repository_set_replacement.patch", @@ -251,13 +247,6 @@ inject_repo(crate, "zstd") use_repo(crate, "argument_comment_lint_crates") bazel_dep(name = "bzip2", version = "1.0.8.bcr.3") -single_version_override( - module_name = "bzip2", - patch_strip = 1, - patches = [ - "//patches:bzip2_windows_stack_args.patch", - ], -) crate.annotation( crate = "bzip2-sys", @@ -277,13 +266,6 @@ crate.annotation( inject_repo(crate, "zlib") bazel_dep(name = "xz", version = "5.4.5.bcr.8") -single_version_override( - module_name = "xz", - patch_strip = 1, - patches = [ - "//patches:xz_windows_stack_args.patch", - ], -) crate.annotation( crate = "lzma-sys", diff --git a/bazel/rules/testing/wine/wine.bzl b/bazel/rules/testing/wine/wine.bzl index 94b0d7a96e13..b1898cc5dae3 100644 --- a/bazel/rules/testing/wine/wine.bzl +++ b/bazel/rules/testing/wine/wine.bzl @@ -10,6 +10,7 @@ def wine_rust_test( windows_binaries, host_binaries = {}, data = [], + tags = [], target_compatible_with = [], **kwargs): """Defines an x86-64 Linux Rust test with a pinned Wine runtime. @@ -36,6 +37,7 @@ def wine_rust_test( windows_binaries: Map from `CARGO_BIN_EXE_*` suffixes to Windows targets. host_binaries: Map from `CARGO_BIN_EXE_*` suffixes to Linux host targets. data: Additional runtime data for the Linux test. + tags: Additional tags for the generated test. target_compatible_with: Additional compatibility constraints. **kwargs: Remaining attributes forwarded to `rust_test`. """ @@ -64,6 +66,7 @@ def wine_rust_test( name = name, data = data + runtime.data, env = runtime.env, + tags = tags + ["windows-gnullvm-incompatible"], target_compatible_with = target_compatible_with + WINE_TEST_TARGET_COMPATIBLE_WITH, **kwargs ) diff --git a/defs.bzl b/defs.bzl index d49b06c267c6..e70aa7ca9bf8 100644 --- a/defs.bzl +++ b/defs.bzl @@ -486,7 +486,10 @@ def codex_rust_crate( ], rustc_env = rustc_env, target_compatible_with = WINDOWS_GNULLVM_INCOMPATIBLE, - tags = test_tags + ["manual"], + tags = test_tags + [ + "manual", + "windows-gnullvm-incompatible", + ], ) workspace_root_test( @@ -500,7 +503,7 @@ def codex_rust_crate( test_bin = ":" + integration_test_binary, workspace_root_marker = "//codex-rs/utils/cargo-bin:repo_root.marker", target_compatible_with = WINDOWS_GNULLVM_INCOMPATIBLE, - tags = test_tags, + tags = test_tags + ["windows-gnullvm-incompatible"], **test_kwargs ) else: @@ -526,7 +529,7 @@ def codex_rust_crate( rustc_env = rustc_env, env = integration_test_cargo_env, target_compatible_with = WINDOWS_GNULLVM_INCOMPATIBLE, - tags = test_tags, + tags = test_tags + ["windows-gnullvm-incompatible"], **test_kwargs ) diff --git a/patches/BUILD.bazel b/patches/BUILD.bazel index 82609770b561..8189f2d058c3 100644 --- a/patches/BUILD.bazel +++ b/patches/BUILD.bazel @@ -3,7 +3,6 @@ exports_files([ "aws-lc-sys_memcmp_check.patch", "aws-lc-sys_windows_msvc_prebuilt_nasm.patch", "aws-lc-sys_windows_msvc_memcmp_probe.patch", - "bzip2_windows_stack_args.patch", "llvm_rusty_v8_custom_libcxx.patch", "llvm_windows_arm64_powl.patch", "llvm_windows_mingw_compat.patch", @@ -26,6 +25,5 @@ exports_files([ "v8_source_portability.patch", "webrtc-sys_hermetic_darwin_sysroot.patch", "windows-link.patch", - "xz_windows_stack_args.patch", "zstd-sys_windows_msvc_include_dirs.patch", ]) diff --git a/patches/aws-lc-sys_windows_msvc_prebuilt_nasm.patch b/patches/aws-lc-sys_windows_msvc_prebuilt_nasm.patch index 76faca025582..c0b129d1ca84 100644 --- a/patches/aws-lc-sys_windows_msvc_prebuilt_nasm.patch +++ b/patches/aws-lc-sys_windows_msvc_prebuilt_nasm.patch @@ -73,12 +73,12 @@ diff --git a/builder/main.rs b/builder/main.rs index 51a9bc1..e714ba4 100644 --- a/builder/main.rs +++ b/builder/main.rs -@@ -723,14 +723,31 @@ fn get_crate_cflags() -> Option { +@@ -723,14 +723,35 @@ fn get_crate_cflags() -> Option { .or(optional_env_optional_crate_target("CFLAGS")) } -+pub(crate) fn is_bazel_windows_msvc_build_script() -> bool { -+ if !target().ends_with("windows-msvc") { ++pub(crate) fn is_bazel_windows_build_script() -> bool { ++ if target_os() != "windows" { + return false; + } + @@ -92,9 +92,13 @@ index 51a9bc1..e714ba4 100644 + .components() + .any(|component| component.as_os_str() == "bazel-out") +} ++ ++pub(crate) fn is_bazel_windows_msvc_build_script() -> bool { ++ target().ends_with("windows-msvc") && is_bazel_windows_build_script() ++} + fn use_prebuilt_nasm() -> bool { -+ let use_prebuilt_for_bazel_windows_msvc = is_bazel_windows_msvc_build_script(); ++ let use_prebuilt_for_bazel_windows = is_bazel_windows_build_script(); target_os() == "windows" && target_arch() == "x86_64" && !is_no_asm() @@ -102,17 +106,17 @@ index 51a9bc1..e714ba4 100644 && Some(false) != allow_prebuilt_nasm() // not prevented by environment && !is_disable_prebuilt_nasm() // not prevented by feature // permitted by environment or by feature -+ && (use_prebuilt_for_bazel_windows_msvc || !test_nasm_command()) ++ && (use_prebuilt_for_bazel_windows || !test_nasm_command()) && (Some(true) == allow_prebuilt_nasm() || is_prebuilt_nasm()) } -@@ -817,8 +834,12 @@ fn main() { +@@ -817,8 +838,12 @@ fn main() { initialize(); prepare_cargo_cfg(); - let manifest_dir = current_dir(); - let manifest_dir = dunce::canonicalize(Path::new(&manifest_dir)).unwrap(); -+ let manifest_dir = if is_bazel_windows_msvc_build_script() { ++ let manifest_dir = if is_bazel_windows_build_script() { + PathBuf::from(cargo_env("CARGO_MANIFEST_DIR")) + } else { + let manifest_dir = current_dir(); diff --git a/patches/bzip2_windows_stack_args.patch b/patches/bzip2_windows_stack_args.patch deleted file mode 100644 index 2786e908be38..000000000000 --- a/patches/bzip2_windows_stack_args.patch +++ /dev/null @@ -1,23 +0,0 @@ -diff --git a/BUILD.bazel b/BUILD.bazel ---- a/BUILD.bazel -+++ b/BUILD.bazel -@@ -28,4 +28,11 @@ cc_library( - defines = [ - "_FILE_OFFSET_BITS=64", - ], -+ copts = select({ -+ "@platforms//os:windows": [ -+ "-fno-stack-protector", -+ "-mno-stack-arg-probe", -+ ], -+ "//conditions:default": [], -+ }), - includes = ["."], -diff --git a/MODULE.bazel b/MODULE.bazel ---- a/MODULE.bazel -+++ b/MODULE.bazel -@@ -4,3 +4,4 @@ module( - ) - - bazel_dep(name = "rules_cc", version = "0.0.10") -+bazel_dep(name = "platforms", version = "1.0.0") diff --git a/patches/rules_rust_windows_exec_msvc_build_script_env.patch b/patches/rules_rust_windows_exec_msvc_build_script_env.patch index 14e114f465e6..38ead6a58066 100644 --- a/patches/rules_rust_windows_exec_msvc_build_script_env.patch +++ b/patches/rules_rust_windows_exec_msvc_build_script_env.patch @@ -1,7 +1,7 @@ diff --git a/cargo/private/cargo_build_script.bzl b/cargo/private/cargo_build_script.bzl --- a/cargo/private/cargo_build_script.bzl +++ b/cargo/private/cargo_build_script.bzl -@@ -144,9 +144,14 @@ def _rewrite_windows_exec_msvc_cc_args(toolchain, args): +@@ -125,9 +125,14 @@ def _rewrite_windows_exec_msvc_cc_args(toolchain, args): if toolchain.target_flag_value != toolchain.exec_triple.str or not toolchain.exec_triple.str.endswith("-pc-windows-msvc"): return args @@ -18,7 +18,7 @@ diff --git a/cargo/private/cargo_build_script.bzl b/cargo/private/cargo_build_sc if skip_next: skip_next = False continue -@@ -161,21 +166,58 @@ def _rewrite_windows_exec_msvc_cc_args(toolchain, args): +@@ -142,21 +147,58 @@ def _rewrite_windows_exec_msvc_cc_args(toolchain, args): if arg == "-nostdlibinc" or arg.startswith("--sysroot"): continue @@ -84,7 +84,7 @@ diff --git a/cargo/private/cargo_build_script.bzl b/cargo/private/cargo_build_sc def get_cc_compile_args_and_env(cc_toolchain, feature_configuration): """Gather cc environment variables from the given `cc_toolchain` -@@ -508,15 +550,23 @@ def _cargo_build_script_impl(ctx): +@@ -487,15 +529,23 @@ def _cargo_build_script_impl(ctx): cc_toolchain, feature_configuration = find_cc_toolchain(ctx) linker, _, link_args, linker_env = get_linker_and_args(ctx, "bin", toolchain, cc_toolchain, feature_configuration, None) env.update(**linker_env) @@ -114,7 +114,7 @@ diff --git a/cargo/private/cargo_build_script.bzl b/cargo/private/cargo_build_sc fallbacks = { "AR": "_fallback_ar", "CC": "_fallback_cc", -@@ -542,36 +592,37 @@ def _cargo_build_script_impl(ctx): +@@ -521,34 +571,35 @@ def _cargo_build_script_impl(ctx): toolchain_tools.append(cc_toolchain.all_files) @@ -136,8 +136,6 @@ diff --git a/cargo/private/cargo_build_script.bzl b/cargo/private/cargo_build_sc - if not env["AR"]: - env["AR"] = cc_toolchain.ar_executable - -- cc_c_args = _strip_stack_protector_for_windows_llvm_mingw(toolchain, cc_c_args) -- cc_cxx_args = _strip_stack_protector_for_windows_llvm_mingw(toolchain, cc_cxx_args) - cc_c_args = _rewrite_windows_exec_msvc_cc_args(toolchain, cc_c_args) - cc_cxx_args = _rewrite_windows_exec_msvc_cc_args(toolchain, cc_cxx_args) - # Populate CFLAGS and CXXFLAGS that cc-rs relies on when building from source, in particular @@ -167,8 +165,6 @@ diff --git a/cargo/private/cargo_build_script.bzl b/cargo/private/cargo_build_sc + if not env["AR"]: + env["AR"] = cc_toolchain.ar_executable + -+ cc_c_args = _strip_stack_protector_for_windows_llvm_mingw(toolchain, cc_c_args) -+ cc_cxx_args = _strip_stack_protector_for_windows_llvm_mingw(toolchain, cc_cxx_args) + cc_c_args = _rewrite_windows_exec_msvc_cc_args(toolchain, cc_c_args) + cc_cxx_args = _rewrite_windows_exec_msvc_cc_args(toolchain, cc_cxx_args) + # Populate CFLAGS and CXXFLAGS that cc-rs relies on when building from source, in particular diff --git a/patches/rules_rust_windows_gnullvm_build_script.patch b/patches/rules_rust_windows_gnullvm_build_script.patch index a1ed9bf14538..dad436cd9f9b 100644 --- a/patches/rules_rust_windows_gnullvm_build_script.patch +++ b/patches/rules_rust_windows_gnullvm_build_script.patch @@ -1,29 +1,10 @@ diff --git a/cargo/private/cargo_build_script.bzl b/cargo/private/cargo_build_script.bzl --- a/cargo/private/cargo_build_script.bzl +++ b/cargo/private/cargo_build_script.bzl -@@ -120,6 +120,63 @@ +@@ -120,6 +120,44 @@ executable = True, ) -+def _strip_stack_protector_for_windows_llvm_mingw(toolchain, args): -+ """Drop stack protector flags unsupported by llvm-mingw build-script probes.""" -+ if "windows-gnullvm" not in toolchain.target_flag_value: -+ return args -+ -+ uses_llvm_mingw = False -+ for arg in args: -+ if "mingw-w64-" in arg: -+ uses_llvm_mingw = True -+ break -+ -+ if not uses_llvm_mingw: -+ return args -+ -+ # llvm-mingw does not ship libssp_nonshared, so forwarding stack-protector -+ # flags through CFLAGS/CXXFLAGS breaks build.rs probe binaries compiled via -+ # cc-rs. -+ return [arg for arg in args if not arg.startswith("-fstack-protector")] -+ +def _rewrite_windows_exec_msvc_cc_args(toolchain, args): + """Translate GNU-flavored cc args when exec-side build scripts target Windows MSVC.""" + if toolchain.target_flag_value != toolchain.exec_triple.str or not toolchain.exec_triple.str.endswith("-pc-windows-msvc"): @@ -65,12 +46,10 @@ diff --git a/cargo/private/cargo_build_script.bzl b/cargo/private/cargo_build_sc def get_cc_compile_args_and_env(cc_toolchain, feature_configuration): """Gather cc environment variables from the given `cc_toolchain` -@@ -503,6 +560,10 @@ +@@ -503,6 +541,8 @@ if not env["AR"]: env["AR"] = cc_toolchain.ar_executable -+ cc_c_args = _strip_stack_protector_for_windows_llvm_mingw(toolchain, cc_c_args) -+ cc_cxx_args = _strip_stack_protector_for_windows_llvm_mingw(toolchain, cc_cxx_args) + cc_c_args = _rewrite_windows_exec_msvc_cc_args(toolchain, cc_c_args) + cc_cxx_args = _rewrite_windows_exec_msvc_cc_args(toolchain, cc_cxx_args) # Populate CFLAGS and CXXFLAGS that cc-rs relies on when building from source, in particular diff --git a/patches/xz_windows_stack_args.patch b/patches/xz_windows_stack_args.patch deleted file mode 100644 index 926fb0a6dc90..000000000000 --- a/patches/xz_windows_stack_args.patch +++ /dev/null @@ -1,14 +0,0 @@ -diff --git a/BUILD.bazel b/BUILD.bazel ---- a/BUILD.bazel -+++ b/BUILD.bazel -@@ -154,6 +154,9 @@ cc_library( - ], - copts = select({ -- "@platforms//os:windows": [], -+ "@platforms//os:windows": [ -+ "-fno-stack-protector", -+ "-mno-stack-arg-probe", -+ ], - "//conditions:default": ["-std=c99"], - }), - defines = select({ diff --git a/scripts/list-bazel-clippy-targets.sh b/scripts/list-bazel-clippy-targets.sh index d12a256d0067..8e05a2fb6bbe 100755 --- a/scripts/list-bazel-clippy-targets.sh +++ b/scripts/list-bazel-clippy-targets.sh @@ -22,23 +22,20 @@ done # Resolve the dynamic targets before printing anything so callers do not # continue with a partial list if `bazel query` fails. Target discovery is # local on all platforms. +manual_rust_test_query='kind("rust_test rule", attr(tags, "manual", //codex-rs/... except //codex-rs/v8-poc/...))' +if [[ "${RUNNER_OS:-}" == "Windows" && $windows_cross_compile -eq 1 ]]; then + manual_rust_test_query="(${manual_rust_test_query}) except attr(tags, \"windows-gnullvm-incompatible\", ${manual_rust_test_query})" +fi + manual_rust_test_targets="$( ./.github/scripts/run-bazel-query-ci.sh \ --output=label \ - -- 'kind("rust_test rule", attr(tags, "manual", //codex-rs/... except //codex-rs/v8-poc/...))' + -- "${manual_rust_test_query}" )" if [[ "${RUNNER_OS:-}" != "Windows" ]]; then # Non-Windows clippy jobs lint the native test binaries; the # Windows-cross binaries exist only for the fast Windows test leg. manual_rust_test_targets="$(printf '%s\n' "${manual_rust_test_targets}" | grep -v -- '-windows-cross-bin$' || true)" -elif [[ $windows_cross_compile -eq 1 ]]; then - # `bazel query` is intentionally pre-analysis and does not remove targets - # made incompatible by `target_compatible_with`. Sharded integration tests - # add native-only manual helpers such as `core-all-test-bin`, plus separate - # `core-all-test-windows-cross-bin` helpers for the Windows cross leg. Keep - # the Windows helpers and unit-test helpers, but do not pass the native-only - # sharded integration helpers as explicit clippy targets. - manual_rust_test_targets="$(printf '%s\n' "${manual_rust_test_targets}" | grep -v -- '-test-bin$' || true)" fi printf '%s\n' \ diff --git a/tools/windows-toolchain/BUILD.bazel b/tools/windows-toolchain/BUILD.bazel new file mode 100644 index 000000000000..a45e8a4f0202 --- /dev/null +++ b/tools/windows-toolchain/BUILD.bazel @@ -0,0 +1,12 @@ +load("@rules_cc//cc:cc_test.bzl", "cc_test") + +cc_test( + name = "stack-protector-probe", + srcs = ["stack_protector_probe.c"], + copts = ["-fstack-protector-all"], + tags = ["manual"], + target_compatible_with = [ + "@platforms//os:windows", + "@rules_rs//rs/experimental/platforms/constraints:windows_gnullvm", + ], +) diff --git a/tools/windows-toolchain/stack_protector_probe.c b/tools/windows-toolchain/stack_protector_probe.c new file mode 100644 index 000000000000..13cdb9261068 --- /dev/null +++ b/tools/windows-toolchain/stack_protector_probe.c @@ -0,0 +1,11 @@ +#if !defined(_WIN32) +#error "this probe must be compiled for Windows" +#endif + +__attribute__((noinline)) static int protected_value(int value) { + volatile char buffer[32] = {0}; + buffer[0] = (char)value; + return buffer[0]; +} + +int main(void) { return protected_value(7) == 7 ? 0 : 1; } From a38819d81f11a12b92220c698ba6d7e4f969ac25 Mon Sep 17 00:00:00 2001 From: Adam Perry Date: Sat, 20 Jun 2026 02:59:13 +0000 Subject: [PATCH 02/10] ci: restore Windows PowerShell for Bazel tests --- .github/scripts/compute-bazel-windows-path.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/scripts/compute-bazel-windows-path.ps1 b/.github/scripts/compute-bazel-windows-path.ps1 index 347d16bfef4c..dc90de8ac3eb 100644 --- a/.github/scripts/compute-bazel-windows-path.ps1 +++ b/.github/scripts/compute-bazel-windows-path.ps1 @@ -36,6 +36,7 @@ $testPathEntries = @( (Join-Path $gitRoot 'bin'), (Join-Path $gitRoot 'usr\bin'), (Join-Path $env:LOCALAPPDATA 'Microsoft\WindowsApps'), + (Join-Path $windowsDir 'System32\WindowsPowerShell\v1.0'), (Join-Path $windowsDir 'System32'), $windowsDir ) From 723775d69a81ed1ea1b99d1a675fe34e81c00d5b Mon Sep 17 00:00:00 2001 From: Adam Perry Date: Sat, 20 Jun 2026 03:44:47 +0000 Subject: [PATCH 03/10] build: provide hermetic Python to Windows Bazel tests --- MODULE.bazel | 8 ++++++ bazel/rules/testing/BUILD.bazel | 8 ++++++ bazel/rules/testing/hermetic_test_python.bzl | 30 ++++++++++++++++++++ defs.bzl | 25 +++++++++++++++- workspace_root_test_launcher.bat.tpl | 20 +++++++++++++ 5 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 bazel/rules/testing/hermetic_test_python.bzl diff --git a/MODULE.bazel b/MODULE.bazel index 1e943122a2ee..eb284fd33938 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -2,8 +2,16 @@ module(name = "codex") bazel_dep(name = "bazel_skylib", version = "1.9.0") bazel_dep(name = "platforms", version = "1.0.0") +bazel_dep(name = "rules_python", version = "1.7.0") bazel_dep(name = "llvm", version = "0.7.9") +# Rust tests exercise hook commands that invoke Python. Keep that runtime in +# the Bazel input closure instead of inheriting hosted Python from the runner. +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.defaults(python_version = "3.11.14") +python.toolchain(python_version = "3.11.14") +use_repo(python, "python_3_11_14") + # Patch hermetic LLVM for Codex's custom libc++ and Windows gnullvm runtime # needs that have not landed upstream. single_version_override( diff --git a/bazel/rules/testing/BUILD.bazel b/bazel/rules/testing/BUILD.bazel index 921cfacc8aa5..eccf5dabc6f8 100644 --- a/bazel/rules/testing/BUILD.bazel +++ b/bazel/rules/testing/BUILD.bazel @@ -1,5 +1,13 @@ +load(":hermetic_test_python.bzl", "hermetic_test_python") + package(default_visibility = ["//visibility:public"]) exports_files([ "foreign_platform_binary.bzl", + "hermetic_test_python.bzl", ]) + +hermetic_test_python( + name = "hermetic_test_python", + testonly = True, +) diff --git a/bazel/rules/testing/hermetic_test_python.bzl b/bazel/rules/testing/hermetic_test_python.bzl new file mode 100644 index 000000000000..3b2fd073ced2 --- /dev/null +++ b/bazel/rules/testing/hermetic_test_python.bzl @@ -0,0 +1,30 @@ +"""Exposes the selected hermetic target Python runtime to Bazel tests.""" + +HermeticTestPythonInfo = provider( + doc = "The interpreter selected from the hermetic target Python toolchain.", + fields = { + "interpreter": "The in-build Python interpreter file.", + }, +) + +def _hermetic_test_python_impl(ctx): + runtime = ctx.toolchains["@rules_python//python:toolchain_type"].py3_runtime + if runtime == None or runtime.interpreter == None: + fail("the hermetic Python toolchain must provide an in-build interpreter") + + runtime_files = depset( + direct = [runtime.interpreter], + transitive = [runtime.files], + ) + return [ + DefaultInfo( + files = runtime_files, + runfiles = ctx.runfiles(transitive_files = runtime_files), + ), + HermeticTestPythonInfo(interpreter = runtime.interpreter), + ] + +hermetic_test_python = rule( + implementation = _hermetic_test_python_impl, + toolchains = ["@rules_python//python:toolchain_type"], +) diff --git a/defs.bzl b/defs.bzl index e70aa7ca9bf8..b4b739e594fb 100644 --- a/defs.bzl +++ b/defs.bzl @@ -3,6 +3,7 @@ load("@crates//:defs.bzl", "all_crate_deps") load("@rules_rust//cargo/private:cargo_build_script_wrapper.bzl", "cargo_build_script") load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library", "rust_proc_macro", "rust_test") load("//bazel/rules/testing:foreign_platform_binary.bzl", "foreign_platform_binary") +load("//bazel/rules/testing:hermetic_test_python.bzl", "HermeticTestPythonInfo") load("//bazel/rules/testing/wine:wine_runtime.bzl", "WINE_TEST_TARGET_COMPATIBLE_WITH", "wine_test_runtime") # Match Cargo's Windows linker behavior so Bazel-built binaries and tests use @@ -75,6 +76,10 @@ def _workspace_root_test_impl(ctx): runfile = _runfile_env_file(runfile_dep) runfiles = runfiles.merge(ctx.runfiles(files = [runfile])) runfiles = runfiles.merge(runfile_dep[DefaultInfo].default_runfiles) + if ctx.attr.hermetic_test_python != None: + test_python = ctx.attr.hermetic_test_python[HermeticTestPythonInfo].interpreter + runfiles = runfiles.merge(ctx.runfiles(files = [test_python])) + runfiles = runfiles.merge(ctx.attr.hermetic_test_python[DefaultInfo].default_runfiles) location_targets = {} for target in ( @@ -112,6 +117,10 @@ def _windows_runfile_env_exports(ctx): runfile = _runfile_env_file(runfile_dep) lines.append('call :resolve_runfile {} "{}"'.format(env_var, _runfile_logical_path(runfile))) lines.append("if errorlevel 1 exit /b 1") + if ctx.attr.hermetic_test_python != None: + test_python = ctx.attr.hermetic_test_python[HermeticTestPythonInfo].interpreter + lines.append('call :resolve_runfile CODEX_BAZEL_TEST_PYTHON "{}"'.format(_runfile_logical_path(test_python))) + lines.append("if errorlevel 1 exit /b 1") return "\n".join(lines) def _runfile_env_file(target): @@ -138,7 +147,7 @@ def _windows_workspace_root_setup(ctx): return """set "INSTA_WORKSPACE_ROOT=%workspace_root%" cd /d "%workspace_root%" || exit /b 1""" -workspace_root_test = rule( +_workspace_root_test = rule( implementation = _workspace_root_test_impl, test = True, toolchains = ["@bazel_tools//tools/test:default_test_toolchain_type"], @@ -150,6 +159,10 @@ workspace_root_test = rule( allow_files = True, ), "env": attr.string_dict(), + "hermetic_test_python": attr.label( + cfg = "target", + providers = [HermeticTestPythonInfo], + ), "runfile_env": attr.label_keyed_string_dict( cfg = "target", ), @@ -177,6 +190,16 @@ workspace_root_test = rule( }, ) +def workspace_root_test(name, **kwargs): + _workspace_root_test( + name = name, + hermetic_test_python = select({ + "@platforms//os:windows": "//bazel/rules/testing:hermetic_test_python", + "//conditions:default": None, + }), + **kwargs + ) + def codex_rust_crate( name, crate_name, diff --git a/workspace_root_test_launcher.bat.tpl b/workspace_root_test_launcher.bat.tpl index 3613b91d76c9..df8eb3cedf17 100644 --- a/workspace_root_test_launcher.bat.tpl +++ b/workspace_root_test_launcher.bat.tpl @@ -12,6 +12,26 @@ if errorlevel 1 exit /b 1 __RUNFILE_ENV_EXPORTS__ +if not defined CODEX_BAZEL_TEST_PYTHON ( + >&2 echo CODEX_BAZEL_TEST_PYTHON must resolve to the hermetic Bazel test interpreter + exit /b 1 +) +if not exist "%CODEX_BAZEL_TEST_PYTHON%" ( + >&2 echo hermetic Bazel test interpreter does not exist: %CODEX_BAZEL_TEST_PYTHON% + exit /b 1 +) +if not defined TEST_TMPDIR ( + >&2 echo TEST_TMPDIR must be set for the hermetic Bazel test Python shim + exit /b 1 +) +set "CODEX_BAZEL_TEST_PYTHON_SHIM_DIR=%TEST_TMPDIR%\codex-bazel-test-python" +if not exist "%CODEX_BAZEL_TEST_PYTHON_SHIM_DIR%" ( + mkdir "%CODEX_BAZEL_TEST_PYTHON_SHIM_DIR%" || exit /b 1 +) +> "%CODEX_BAZEL_TEST_PYTHON_SHIM_DIR%\python3.cmd" echo @echo off +>> "%CODEX_BAZEL_TEST_PYTHON_SHIM_DIR%\python3.cmd" echo "%CODEX_BAZEL_TEST_PYTHON%" %%* +set "PATH=%CODEX_BAZEL_TEST_PYTHON_SHIM_DIR%;%PATH%" + __WORKSPACE_ROOT_SETUP__ set "TOTAL_SHARDS=%RULES_RUST_TEST_TOTAL_SHARDS%" From d475589f6cb2c717127acf697651cab9b70f3521 Mon Sep 17 00:00:00 2001 From: Adam Perry Date: Sat, 20 Jun 2026 04:01:37 +0000 Subject: [PATCH 04/10] build: resolve self-mapped Windows runfiles --- workspace_root_test_launcher.bat.tpl | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/workspace_root_test_launcher.bat.tpl b/workspace_root_test_launcher.bat.tpl index df8eb3cedf17..fc04f3138de3 100644 --- a/workspace_root_test_launcher.bat.tpl +++ b/workspace_root_test_launcher.bat.tpl @@ -145,13 +145,33 @@ if defined manifest if exist "%manifest%" ( rem Read the manifest directly instead of shelling out to findstr. In the rem GitHub Windows runner, the nested `findstr` path produced rem `FINDSTR: Cannot open D:MANIFEST`, which then broke runfile resolution for - rem Bazel tests even though the manifest file was present. + rem Bazel tests even though the manifest file was present. A one-field + rem manifest entry self-maps its logical link; accept that form only when the + rem link exists, and never report success with an empty resolved path. for /f "usebackq tokens=1,* delims= " %%A in ("%manifest%") do ( if "%%A"=="%logical_path%" ( - endlocal & set "%~1=%%B" & exit /b 0 + if not "%%B"=="" ( + endlocal & set "%~1=%%B" & exit /b 0 + ) + if exist "!native_logical_path!" ( + for %%P in ("!native_logical_path!") do ( + endlocal + set "%~1=%%~fP" + exit /b 0 + ) + ) ) if "%%A"=="%workspace_logical_path%" ( - endlocal & set "%~1=%%B" & exit /b 0 + if not "%%B"=="" ( + endlocal & set "%~1=%%B" & exit /b 0 + ) + if exist "!native_workspace_logical_path!" ( + for %%P in ("!native_workspace_logical_path!") do ( + endlocal + set "%~1=%%~fP" + exit /b 0 + ) + ) ) ) ) From 71b8ed97203dd61c2b38cb2dfb06b039240ef2b2 Mon Sep 17 00:00:00 2001 From: Adam Perry Date: Sat, 20 Jun 2026 04:16:34 +0000 Subject: [PATCH 05/10] build: preserve resolved Windows runfile paths --- workspace_root_test_launcher.bat.tpl | 38 ++++++++++++++++------------ 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/workspace_root_test_launcher.bat.tpl b/workspace_root_test_launcher.bat.tpl index fc04f3138de3..611c247026c0 100644 --- a/workspace_root_test_launcher.bat.tpl +++ b/workspace_root_test_launcher.bat.tpl @@ -128,10 +128,12 @@ for %%R in ("%RUNFILES_DIR%" "%TEST_SRCDIR%") do ( set "runfiles_root=%%~R" if defined runfiles_root ( if exist "!runfiles_root!\!native_logical_path!" ( - endlocal & set "%~1=!runfiles_root!\!native_logical_path!" & exit /b 0 + set "resolved_runfile=!runfiles_root!\!native_logical_path!" + goto :resolve_runfile_found ) if exist "!runfiles_root!\!native_workspace_logical_path!" ( - endlocal & set "%~1=!runfiles_root!\!native_workspace_logical_path!" & exit /b 0 + set "resolved_runfile=!runfiles_root!\!native_workspace_logical_path!" + goto :resolve_runfile_found ) ) ) @@ -150,31 +152,35 @@ if defined manifest if exist "%manifest%" ( rem link exists, and never report success with an empty resolved path. for /f "usebackq tokens=1,* delims= " %%A in ("%manifest%") do ( if "%%A"=="%logical_path%" ( - if not "%%B"=="" ( - endlocal & set "%~1=%%B" & exit /b 0 + if not "%%B"=="" if exist "%%B" ( + set "resolved_runfile=%%B" + goto :resolve_runfile_found ) if exist "!native_logical_path!" ( - for %%P in ("!native_logical_path!") do ( - endlocal - set "%~1=%%~fP" - exit /b 0 - ) + set "resolved_runfile=!native_logical_path!" + goto :resolve_runfile_found ) ) if "%%A"=="%workspace_logical_path%" ( - if not "%%B"=="" ( - endlocal & set "%~1=%%B" & exit /b 0 + if not "%%B"=="" if exist "%%B" ( + set "resolved_runfile=%%B" + goto :resolve_runfile_found ) if exist "!native_workspace_logical_path!" ( - for %%P in ("!native_workspace_logical_path!") do ( - endlocal - set "%~1=%%~fP" - exit /b 0 - ) + set "resolved_runfile=!native_workspace_logical_path!" + goto :resolve_runfile_found ) ) ) ) +:resolve_runfile_not_found >&2 echo failed to resolve runfile: %logical_path% endlocal & exit /b 1 + +:resolve_runfile_found +if not defined resolved_runfile goto :resolve_runfile_not_found +if not exist "!resolved_runfile!" goto :resolve_runfile_not_found +for %%P in ("!resolved_runfile!") do set "resolved_runfile=%%~fP" +if not defined resolved_runfile goto :resolve_runfile_not_found +endlocal & set "%~1=%resolved_runfile%" & exit /b 0 From 3e4edd748666b635d950a0e1d5014f753e5c4071 Mon Sep 17 00:00:00 2001 From: Adam Perry Date: Sat, 20 Jun 2026 04:50:21 +0000 Subject: [PATCH 06/10] build: wrap hermetic Windows test Python --- bazel/rules/testing/BUILD.bazel | 9 +++++ bazel/rules/testing/hermetic_test_python.bzl | 41 +++++++++++++------- bazel/rules/testing/hermetic_test_python.py | 10 +++++ defs.bzl | 4 +- patches/llvm_windows_mingw_compat.patch | 24 ++++++++++-- workspace_root_test_launcher.bat.tpl | 36 +++-------------- 6 files changed, 74 insertions(+), 50 deletions(-) create mode 100644 bazel/rules/testing/hermetic_test_python.py diff --git a/bazel/rules/testing/BUILD.bazel b/bazel/rules/testing/BUILD.bazel index eccf5dabc6f8..e476210d4688 100644 --- a/bazel/rules/testing/BUILD.bazel +++ b/bazel/rules/testing/BUILD.bazel @@ -1,3 +1,4 @@ +load("@rules_python//python:defs.bzl", "py_binary") load(":hermetic_test_python.bzl", "hermetic_test_python") package(default_visibility = ["//visibility:public"]) @@ -7,6 +8,14 @@ exports_files([ "hermetic_test_python.bzl", ]) +py_binary( + name = "_hermetic_test_python", + testonly = True, + srcs = ["hermetic_test_python.py"], + main = "hermetic_test_python.py", + visibility = ["//visibility:private"], +) + hermetic_test_python( name = "hermetic_test_python", testonly = True, diff --git a/bazel/rules/testing/hermetic_test_python.bzl b/bazel/rules/testing/hermetic_test_python.bzl index 3b2fd073ced2..01361c72dc67 100644 --- a/bazel/rules/testing/hermetic_test_python.bzl +++ b/bazel/rules/testing/hermetic_test_python.bzl @@ -1,30 +1,43 @@ -"""Exposes the selected hermetic target Python runtime to Bazel tests.""" +"""Exposes a fixed hermetic Python forwarder to Bazel tests.""" HermeticTestPythonInfo = provider( - doc = "The interpreter selected from the hermetic target Python toolchain.", + doc = "The fixed forwarding executable backed by the hermetic target Python toolchain.", fields = { - "interpreter": "The in-build Python interpreter file.", + "executable": "The main-repository Python forwarding executable.", }, ) def _hermetic_test_python_impl(ctx): - runtime = ctx.toolchains["@rules_python//python:toolchain_type"].py3_runtime - if runtime == None or runtime.interpreter == None: - fail("the hermetic Python toolchain must provide an in-build interpreter") + launcher_info = ctx.attr._launcher[DefaultInfo] + files_to_run = launcher_info.files_to_run + executable = files_to_run.executable + if executable == None: + fail("the hermetic Python forwarder must provide an executable") + + support_files = [executable] + if files_to_run.runfiles_manifest != None: + support_files.append(files_to_run.runfiles_manifest) + if files_to_run.repo_mapping_manifest != None: + support_files.append(files_to_run.repo_mapping_manifest) - runtime_files = depset( - direct = [runtime.interpreter], - transitive = [runtime.files], - ) return [ DefaultInfo( - files = runtime_files, - runfiles = ctx.runfiles(transitive_files = runtime_files), + files = launcher_info.files, + runfiles = ctx.runfiles( + files = support_files, + transitive_files = launcher_info.files, + ).merge(launcher_info.default_runfiles), ), - HermeticTestPythonInfo(interpreter = runtime.interpreter), + HermeticTestPythonInfo(executable = executable), ] hermetic_test_python = rule( implementation = _hermetic_test_python_impl, - toolchains = ["@rules_python//python:toolchain_type"], + attrs = { + "_launcher": attr.label( + cfg = "target", + default = "//bazel/rules/testing:_hermetic_test_python", + executable = True, + ), + }, ) diff --git a/bazel/rules/testing/hermetic_test_python.py b/bazel/rules/testing/hermetic_test_python.py new file mode 100644 index 000000000000..6b1b7aad4099 --- /dev/null +++ b/bazel/rules/testing/hermetic_test_python.py @@ -0,0 +1,10 @@ +import subprocess +import sys + + +def main() -> int: + return subprocess.call([sys.executable, *sys.argv[1:]]) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/defs.bzl b/defs.bzl index b4b739e594fb..9d0860642c14 100644 --- a/defs.bzl +++ b/defs.bzl @@ -77,7 +77,7 @@ def _workspace_root_test_impl(ctx): runfiles = runfiles.merge(ctx.runfiles(files = [runfile])) runfiles = runfiles.merge(runfile_dep[DefaultInfo].default_runfiles) if ctx.attr.hermetic_test_python != None: - test_python = ctx.attr.hermetic_test_python[HermeticTestPythonInfo].interpreter + test_python = ctx.attr.hermetic_test_python[HermeticTestPythonInfo].executable runfiles = runfiles.merge(ctx.runfiles(files = [test_python])) runfiles = runfiles.merge(ctx.attr.hermetic_test_python[DefaultInfo].default_runfiles) @@ -118,7 +118,7 @@ def _windows_runfile_env_exports(ctx): lines.append('call :resolve_runfile {} "{}"'.format(env_var, _runfile_logical_path(runfile))) lines.append("if errorlevel 1 exit /b 1") if ctx.attr.hermetic_test_python != None: - test_python = ctx.attr.hermetic_test_python[HermeticTestPythonInfo].interpreter + test_python = ctx.attr.hermetic_test_python[HermeticTestPythonInfo].executable lines.append('call :resolve_runfile CODEX_BAZEL_TEST_PYTHON "{}"'.format(_runfile_logical_path(test_python))) lines.append("if errorlevel 1 exit /b 1") return "\n".join(lines) diff --git a/patches/llvm_windows_mingw_compat.patch b/patches/llvm_windows_mingw_compat.patch index 66cf4e7b3446..7f5589c2961b 100644 --- a/patches/llvm_windows_mingw_compat.patch +++ b/patches/llvm_windows_mingw_compat.patch @@ -1,6 +1,24 @@ -# What: provide MinGW compatibility archives required by Windows gnullvm links. -# Why: clang and prebuilt MSVC archives can request these names even though the -# hermetic MinGW runtime does not publish them by default. +# What: provide MinGW compile and link compatibility required by Windows gnullvm. +# Why: Bazel's Windows launcher needs the UCRT secure-random declaration, while +# clang and prebuilt MSVC archives request libraries the hermetic runtime does +# not publish by default. + +diff --git a/toolchain/args/BUILD.bazel b/toolchain/args/BUILD.bazel +--- a/toolchain/args/BUILD.bazel ++++ b/toolchain/args/BUILD.bazel +@@ -765,5 +765,12 @@ cc_args( + # provided as part of the compiler toolchain repository. + "-nostdlibinc", +- ], ++ ] + select({ ++ # MinGW exposes rand_s only when this UCRT feature macro is defined. ++ # Bazel's native Windows launcher uses rand_s unconditionally, so make ++ # the declaration available to every hermetic Windows compile. ++ "//platforms/config:windows_aarch64": ["-D_CRT_RAND_S="], ++ "//platforms/config:windows_x86_64": ["-D_CRT_RAND_S="], ++ "//conditions:default": [], ++ }), + ) diff --git a/runtimes/mingw/BUILD.bazel b/runtimes/mingw/BUILD.bazel --- a/runtimes/mingw/BUILD.bazel diff --git a/workspace_root_test_launcher.bat.tpl b/workspace_root_test_launcher.bat.tpl index 611c247026c0..df8eb3cedf17 100644 --- a/workspace_root_test_launcher.bat.tpl +++ b/workspace_root_test_launcher.bat.tpl @@ -128,12 +128,10 @@ for %%R in ("%RUNFILES_DIR%" "%TEST_SRCDIR%") do ( set "runfiles_root=%%~R" if defined runfiles_root ( if exist "!runfiles_root!\!native_logical_path!" ( - set "resolved_runfile=!runfiles_root!\!native_logical_path!" - goto :resolve_runfile_found + endlocal & set "%~1=!runfiles_root!\!native_logical_path!" & exit /b 0 ) if exist "!runfiles_root!\!native_workspace_logical_path!" ( - set "resolved_runfile=!runfiles_root!\!native_workspace_logical_path!" - goto :resolve_runfile_found + endlocal & set "%~1=!runfiles_root!\!native_workspace_logical_path!" & exit /b 0 ) ) ) @@ -147,40 +145,16 @@ if defined manifest if exist "%manifest%" ( rem Read the manifest directly instead of shelling out to findstr. In the rem GitHub Windows runner, the nested `findstr` path produced rem `FINDSTR: Cannot open D:MANIFEST`, which then broke runfile resolution for - rem Bazel tests even though the manifest file was present. A one-field - rem manifest entry self-maps its logical link; accept that form only when the - rem link exists, and never report success with an empty resolved path. + rem Bazel tests even though the manifest file was present. for /f "usebackq tokens=1,* delims= " %%A in ("%manifest%") do ( if "%%A"=="%logical_path%" ( - if not "%%B"=="" if exist "%%B" ( - set "resolved_runfile=%%B" - goto :resolve_runfile_found - ) - if exist "!native_logical_path!" ( - set "resolved_runfile=!native_logical_path!" - goto :resolve_runfile_found - ) + endlocal & set "%~1=%%B" & exit /b 0 ) if "%%A"=="%workspace_logical_path%" ( - if not "%%B"=="" if exist "%%B" ( - set "resolved_runfile=%%B" - goto :resolve_runfile_found - ) - if exist "!native_workspace_logical_path!" ( - set "resolved_runfile=!native_workspace_logical_path!" - goto :resolve_runfile_found - ) + endlocal & set "%~1=%%B" & exit /b 0 ) ) ) -:resolve_runfile_not_found >&2 echo failed to resolve runfile: %logical_path% endlocal & exit /b 1 - -:resolve_runfile_found -if not defined resolved_runfile goto :resolve_runfile_not_found -if not exist "!resolved_runfile!" goto :resolve_runfile_not_found -for %%P in ("!resolved_runfile!") do set "resolved_runfile=%%~fP" -if not defined resolved_runfile goto :resolve_runfile_not_found -endlocal & set "%~1=%resolved_runfile%" & exit /b 0 From 60ed308bf26e091353686667395c8342cc6e7380 Mon Sep 17 00:00:00 2001 From: Adam Perry Date: Sat, 20 Jun 2026 05:04:50 +0000 Subject: [PATCH 07/10] build: use fixed execpath for Windows test Python --- defs.bzl | 11 +++++++---- workspace_root_test_launcher.bat.tpl | 6 ++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/defs.bzl b/defs.bzl index 9d0860642c14..544efd89e75e 100644 --- a/defs.bzl +++ b/defs.bzl @@ -55,6 +55,7 @@ def _workspace_root_test_impl(ctx): workspace_root_marker = ctx.file.workspace_root_marker launcher_template = ctx.file._windows_launcher_template if is_windows else ctx.file._bash_launcher_template runfile_env_exports = _windows_runfile_env_exports(ctx) if is_windows else _bash_runfile_env_exports(ctx) + test_python_exec_path = _windows_test_python_exec_path(ctx) if is_windows else "" workspace_root_setup = _windows_workspace_root_setup(ctx) if is_windows else _bash_workspace_root_setup(ctx) ctx.actions.expand_template( template = launcher_template, @@ -63,6 +64,7 @@ def _workspace_root_test_impl(ctx): substitutions = { "__RUNFILE_ENV_EXPORTS__": runfile_env_exports, "__TEST_BIN__": test_bin.short_path, + "__TEST_PYTHON_EXEC_PATH__": test_python_exec_path, "__WORKSPACE_ROOT_SETUP__": workspace_root_setup, "__WORKSPACE_ROOT_MARKER__": workspace_root_marker.short_path, }, @@ -117,12 +119,13 @@ def _windows_runfile_env_exports(ctx): runfile = _runfile_env_file(runfile_dep) lines.append('call :resolve_runfile {} "{}"'.format(env_var, _runfile_logical_path(runfile))) lines.append("if errorlevel 1 exit /b 1") - if ctx.attr.hermetic_test_python != None: - test_python = ctx.attr.hermetic_test_python[HermeticTestPythonInfo].executable - lines.append('call :resolve_runfile CODEX_BAZEL_TEST_PYTHON "{}"'.format(_runfile_logical_path(test_python))) - lines.append("if errorlevel 1 exit /b 1") return "\n".join(lines) +def _windows_test_python_exec_path(ctx): + if ctx.attr.hermetic_test_python == None: + fail("Windows tests must provide the fixed hermetic Python tool") + return ctx.attr.hermetic_test_python[HermeticTestPythonInfo].executable.path + def _runfile_env_file(target): executable = target[DefaultInfo].files_to_run.executable if executable != None: diff --git a/workspace_root_test_launcher.bat.tpl b/workspace_root_test_launcher.bat.tpl index df8eb3cedf17..eeed72074618 100644 --- a/workspace_root_test_launcher.bat.tpl +++ b/workspace_root_test_launcher.bat.tpl @@ -12,10 +12,8 @@ if errorlevel 1 exit /b 1 __RUNFILE_ENV_EXPORTS__ -if not defined CODEX_BAZEL_TEST_PYTHON ( - >&2 echo CODEX_BAZEL_TEST_PYTHON must resolve to the hermetic Bazel test interpreter - exit /b 1 -) +set "CODEX_BAZEL_TEST_PYTHON=__TEST_PYTHON_EXEC_PATH__" +for %%P in ("%CODEX_BAZEL_TEST_PYTHON%") do set "CODEX_BAZEL_TEST_PYTHON=%%~fP" if not exist "%CODEX_BAZEL_TEST_PYTHON%" ( >&2 echo hermetic Bazel test interpreter does not exist: %CODEX_BAZEL_TEST_PYTHON% exit /b 1 From f4d0ca41c804e62c471f9f020bfc11c1dfa69c4f Mon Sep 17 00:00:00 2001 From: Adam Perry Date: Sat, 20 Jun 2026 05:16:15 +0000 Subject: [PATCH 08/10] build: anchor hermetic test Python to launcher --- defs.bzl | 24 ++++++++++++++++++++---- workspace_root_test_launcher.bat.tpl | 2 +- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/defs.bzl b/defs.bzl index 544efd89e75e..7b423b632944 100644 --- a/defs.bzl +++ b/defs.bzl @@ -55,7 +55,7 @@ def _workspace_root_test_impl(ctx): workspace_root_marker = ctx.file.workspace_root_marker launcher_template = ctx.file._windows_launcher_template if is_windows else ctx.file._bash_launcher_template runfile_env_exports = _windows_runfile_env_exports(ctx) if is_windows else _bash_runfile_env_exports(ctx) - test_python_exec_path = _windows_test_python_exec_path(ctx) if is_windows else "" + test_python_relative_path = _windows_test_python_relative_path(ctx, launcher) if is_windows else "" workspace_root_setup = _windows_workspace_root_setup(ctx) if is_windows else _bash_workspace_root_setup(ctx) ctx.actions.expand_template( template = launcher_template, @@ -64,7 +64,7 @@ def _workspace_root_test_impl(ctx): substitutions = { "__RUNFILE_ENV_EXPORTS__": runfile_env_exports, "__TEST_BIN__": test_bin.short_path, - "__TEST_PYTHON_EXEC_PATH__": test_python_exec_path, + "__TEST_PYTHON_RELATIVE_PATH__": test_python_relative_path, "__WORKSPACE_ROOT_SETUP__": workspace_root_setup, "__WORKSPACE_ROOT_MARKER__": workspace_root_marker.short_path, }, @@ -121,10 +121,26 @@ def _windows_runfile_env_exports(ctx): lines.append("if errorlevel 1 exit /b 1") return "\n".join(lines) -def _windows_test_python_exec_path(ctx): +def _windows_test_python_relative_path(ctx, launcher): if ctx.attr.hermetic_test_python == None: fail("Windows tests must provide the fixed hermetic Python tool") - return ctx.attr.hermetic_test_python[HermeticTestPythonInfo].executable.path + + executable = ctx.attr.hermetic_test_python[HermeticTestPythonInfo].executable + launcher_dir_parts = launcher.dirname.split("/") + executable_path_parts = executable.path.split("/") + common_prefix_len = 0 + prefix_matches = True + for index in range(min(len(launcher_dir_parts), len(executable_path_parts))): + if prefix_matches and launcher_dir_parts[index] == executable_path_parts[index]: + common_prefix_len += 1 + else: + prefix_matches = False + + relative_path_parts = ( + [".."] * (len(launcher_dir_parts) - common_prefix_len) + + executable_path_parts[common_prefix_len:] + ) + return "\\".join(relative_path_parts) def _runfile_env_file(target): executable = target[DefaultInfo].files_to_run.executable diff --git a/workspace_root_test_launcher.bat.tpl b/workspace_root_test_launcher.bat.tpl index eeed72074618..d9b69b788d8f 100644 --- a/workspace_root_test_launcher.bat.tpl +++ b/workspace_root_test_launcher.bat.tpl @@ -12,7 +12,7 @@ if errorlevel 1 exit /b 1 __RUNFILE_ENV_EXPORTS__ -set "CODEX_BAZEL_TEST_PYTHON=__TEST_PYTHON_EXEC_PATH__" +set "CODEX_BAZEL_TEST_PYTHON=%~dp0__TEST_PYTHON_RELATIVE_PATH__" for %%P in ("%CODEX_BAZEL_TEST_PYTHON%") do set "CODEX_BAZEL_TEST_PYTHON=%%~fP" if not exist "%CODEX_BAZEL_TEST_PYTHON%" ( >&2 echo hermetic Bazel test interpreter does not exist: %CODEX_BAZEL_TEST_PYTHON% From 6084f50b451c140806a805e323dd5020a8b76916 Mon Sep 17 00:00:00 2001 From: Adam Perry Date: Sat, 20 Jun 2026 05:54:29 +0000 Subject: [PATCH 09/10] build: use direct hermetic Python runtime --- bazel/rules/testing/BUILD.bazel | 9 ----- bazel/rules/testing/hermetic_test_python.bzl | 37 +++++++------------- bazel/rules/testing/hermetic_test_python.py | 10 ------ patches/llvm_windows_mingw_compat.patch | 24 ++----------- 4 files changed, 16 insertions(+), 64 deletions(-) delete mode 100644 bazel/rules/testing/hermetic_test_python.py diff --git a/bazel/rules/testing/BUILD.bazel b/bazel/rules/testing/BUILD.bazel index e476210d4688..eccf5dabc6f8 100644 --- a/bazel/rules/testing/BUILD.bazel +++ b/bazel/rules/testing/BUILD.bazel @@ -1,4 +1,3 @@ -load("@rules_python//python:defs.bzl", "py_binary") load(":hermetic_test_python.bzl", "hermetic_test_python") package(default_visibility = ["//visibility:public"]) @@ -8,14 +7,6 @@ exports_files([ "hermetic_test_python.bzl", ]) -py_binary( - name = "_hermetic_test_python", - testonly = True, - srcs = ["hermetic_test_python.py"], - main = "hermetic_test_python.py", - visibility = ["//visibility:private"], -) - hermetic_test_python( name = "hermetic_test_python", testonly = True, diff --git a/bazel/rules/testing/hermetic_test_python.bzl b/bazel/rules/testing/hermetic_test_python.bzl index 01361c72dc67..ef104168ac07 100644 --- a/bazel/rules/testing/hermetic_test_python.bzl +++ b/bazel/rules/testing/hermetic_test_python.bzl @@ -1,43 +1,32 @@ -"""Exposes a fixed hermetic Python forwarder to Bazel tests.""" +"""Exposes the fixed hermetic target Python runtime to Bazel tests.""" HermeticTestPythonInfo = provider( - doc = "The fixed forwarding executable backed by the hermetic target Python toolchain.", + doc = "The fixed interpreter from the hermetic target Python toolchain.", fields = { - "executable": "The main-repository Python forwarding executable.", + "executable": "The hermetic target Python interpreter.", }, ) def _hermetic_test_python_impl(ctx): - launcher_info = ctx.attr._launcher[DefaultInfo] - files_to_run = launcher_info.files_to_run - executable = files_to_run.executable + runtime = ctx.toolchains["@rules_python//python:toolchain_type"].py3_runtime + if runtime == None: + fail("the hermetic Python toolchain must provide a Python 3 runtime") + + executable = runtime.interpreter if executable == None: - fail("the hermetic Python forwarder must provide an executable") + fail("the hermetic Python 3 runtime must provide an interpreter file") - support_files = [executable] - if files_to_run.runfiles_manifest != None: - support_files.append(files_to_run.runfiles_manifest) - if files_to_run.repo_mapping_manifest != None: - support_files.append(files_to_run.repo_mapping_manifest) + files = depset([executable], transitive = [runtime.files]) return [ DefaultInfo( - files = launcher_info.files, - runfiles = ctx.runfiles( - files = support_files, - transitive_files = launcher_info.files, - ).merge(launcher_info.default_runfiles), + files = files, + runfiles = ctx.runfiles(transitive_files = files), ), HermeticTestPythonInfo(executable = executable), ] hermetic_test_python = rule( implementation = _hermetic_test_python_impl, - attrs = { - "_launcher": attr.label( - cfg = "target", - default = "//bazel/rules/testing:_hermetic_test_python", - executable = True, - ), - }, + toolchains = ["@rules_python//python:toolchain_type"], ) diff --git a/bazel/rules/testing/hermetic_test_python.py b/bazel/rules/testing/hermetic_test_python.py deleted file mode 100644 index 6b1b7aad4099..000000000000 --- a/bazel/rules/testing/hermetic_test_python.py +++ /dev/null @@ -1,10 +0,0 @@ -import subprocess -import sys - - -def main() -> int: - return subprocess.call([sys.executable, *sys.argv[1:]]) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/patches/llvm_windows_mingw_compat.patch b/patches/llvm_windows_mingw_compat.patch index 7f5589c2961b..ca0d9b23ed9b 100644 --- a/patches/llvm_windows_mingw_compat.patch +++ b/patches/llvm_windows_mingw_compat.patch @@ -1,24 +1,6 @@ -# What: provide MinGW compile and link compatibility required by Windows gnullvm. -# Why: Bazel's Windows launcher needs the UCRT secure-random declaration, while -# clang and prebuilt MSVC archives request libraries the hermetic runtime does -# not publish by default. - -diff --git a/toolchain/args/BUILD.bazel b/toolchain/args/BUILD.bazel ---- a/toolchain/args/BUILD.bazel -+++ b/toolchain/args/BUILD.bazel -@@ -765,5 +765,12 @@ cc_args( - # provided as part of the compiler toolchain repository. - "-nostdlibinc", -- ], -+ ] + select({ -+ # MinGW exposes rand_s only when this UCRT feature macro is defined. -+ # Bazel's native Windows launcher uses rand_s unconditionally, so make -+ # the declaration available to every hermetic Windows compile. -+ "//platforms/config:windows_aarch64": ["-D_CRT_RAND_S="], -+ "//platforms/config:windows_x86_64": ["-D_CRT_RAND_S="], -+ "//conditions:default": [], -+ }), - ) +# What: provide MinGW link compatibility required by Windows gnullvm. +# Why: Clang and prebuilt MSVC archives request libraries the hermetic runtime +# does not publish by default. diff --git a/runtimes/mingw/BUILD.bazel b/runtimes/mingw/BUILD.bazel --- a/runtimes/mingw/BUILD.bazel From 7835d1dc10b435bf94201586b68f6eeef0dae480 Mon Sep 17 00:00:00 2001 From: Adam Perry Date: Sat, 20 Jun 2026 06:35:47 +0000 Subject: [PATCH 10/10] build: expose fixed Python alias to Windows tests --- workspace_root_test_launcher.bat.tpl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/workspace_root_test_launcher.bat.tpl b/workspace_root_test_launcher.bat.tpl index d9b69b788d8f..95b73e0d3c7d 100644 --- a/workspace_root_test_launcher.bat.tpl +++ b/workspace_root_test_launcher.bat.tpl @@ -28,6 +28,8 @@ if not exist "%CODEX_BAZEL_TEST_PYTHON_SHIM_DIR%" ( ) > "%CODEX_BAZEL_TEST_PYTHON_SHIM_DIR%\python3.cmd" echo @echo off >> "%CODEX_BAZEL_TEST_PYTHON_SHIM_DIR%\python3.cmd" echo "%CODEX_BAZEL_TEST_PYTHON%" %%* +> "%CODEX_BAZEL_TEST_PYTHON_SHIM_DIR%\python.cmd" echo @echo off +>> "%CODEX_BAZEL_TEST_PYTHON_SHIM_DIR%\python.cmd" echo "%CODEX_BAZEL_TEST_PYTHON%" %%* set "PATH=%CODEX_BAZEL_TEST_PYTHON_SHIM_DIR%;%PATH%" __WORKSPACE_ROOT_SETUP__