From 0bdec6ee6be200385f95025bb1912e31fe681538 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 21 May 2026 08:54:42 -0700 Subject: [PATCH 1/7] npm: remove legacy package artifact synthesis --- .github/workflows/ci.yml | 11 +- codex-cli/scripts/install_native_deps.py | 378 +----------------- codex-cli/scripts/test_install_native_deps.py | 76 ++++ scripts/stage_npm_packages.py | 14 - 4 files changed, 90 insertions(+), 389 deletions(-) create mode 100644 codex-cli/scripts/test_install_native_deps.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 25dff134a77..b1ee1395e10 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,9 +42,6 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - # stage_npm_packages.py requires DotSlash when staging releases. - - uses: facebook/install-dotslash@1e4e7b3e07eaca387acb98f1d4720e0bee8dbb6a # v2 - - name: Stage npm package id: stage_npm_package env: @@ -55,17 +52,13 @@ jobs: # cross-platform native payload required by the npm package layout. # Passing the workflow URL directly avoids relying on old rust-v* # branches remaining discoverable via `gh run list --branch ...`. - CODEX_VERSION=0.125.0 - WORKFLOW_URL="https://github.com/openai/codex/actions/runs/26131514935" + CODEX_VERSION=0.133.0-alpha.4 + WORKFLOW_URL="https://github.com/openai/codex/actions/runs/26201494185" OUTPUT_DIR="${RUNNER_TEMP}" - # This reused workflow predates codex-package archive artifacts, so - # CI synthesizes the package layout from the older per-binary - # artifacts. Release staging must use real package archives. python3 ./scripts/stage_npm_packages.py \ --release-version "$CODEX_VERSION" \ --workflow-url "$WORKFLOW_URL" \ --package codex \ - --allow-legacy-codex-package \ --output-dir "$OUTPUT_DIR" PACK_OUTPUT="${OUTPUT_DIR}/codex-npm-${CODEX_VERSION}.tgz" echo "pack_output=$PACK_OUTPUT" >> "$GITHUB_OUTPUT" diff --git a/codex-cli/scripts/install_native_deps.py b/codex-cli/scripts/install_native_deps.py index de157334cd0..cd6c9063457 100755 --- a/codex-cli/scripts/install_native_deps.py +++ b/codex-cli/scripts/install_native_deps.py @@ -3,27 +3,20 @@ import argparse from contextlib import contextmanager -import json import os import shutil import subprocess import tarfile import tempfile -import zipfile from dataclasses import dataclass from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path -import sys -from typing import Iterable, Sequence -from urllib.parse import urlparse -from urllib.request import urlopen +from typing import Sequence SCRIPT_DIR = Path(__file__).resolve().parent CODEX_CLI_ROOT = SCRIPT_DIR.parent -REPO_ROOT = CODEX_CLI_ROOT.parent -DEFAULT_WORKFLOW_URL = "https://github.com/openai/codex/actions/runs/26131514935" # rust-v0.132.0 +DEFAULT_WORKFLOW_URL = "https://github.com/openai/codex/actions/runs/26201494185" # rust-v0.133.0-alpha.4 VENDOR_DIR_NAME = "vendor" -RG_MANIFEST = REPO_ROOT / "scripts" / "codex_package" / "rg" BINARY_TARGETS = ( "x86_64-unknown-linux-musl", "aarch64-unknown-linux-musl", @@ -40,57 +33,16 @@ class BinaryComponent: artifact_prefix: str # matches the artifact filename prefix (e.g. codex-.zst) dest_dir: str # directory under vendor// where the binary is installed binary_basename: str # executable name inside dest_dir (before optional .exe) - targets: tuple[str, ...] | None = None # limit installation to specific targets -WINDOWS_TARGETS = tuple(target for target in BINARY_TARGETS if "windows" in target) -LINUX_TARGETS = tuple(target for target in BINARY_TARGETS if "linux" in target) - BINARY_COMPONENTS = { - "bwrap": BinaryComponent( - artifact_prefix="bwrap", - dest_dir="codex-resources", - binary_basename="bwrap", - targets=LINUX_TARGETS, - ), - "codex": BinaryComponent( - artifact_prefix="codex", - dest_dir="codex", - binary_basename="codex", - ), "codex-responses-api-proxy": BinaryComponent( artifact_prefix="codex-responses-api-proxy", dest_dir="codex-responses-api-proxy", binary_basename="codex-responses-api-proxy", ), - "codex-windows-sandbox-setup": BinaryComponent( - artifact_prefix="codex-windows-sandbox-setup", - dest_dir="codex", - binary_basename="codex-windows-sandbox-setup", - targets=WINDOWS_TARGETS, - ), - "codex-command-runner": BinaryComponent( - artifact_prefix="codex-command-runner", - dest_dir="codex", - binary_basename="codex-command-runner", - targets=WINDOWS_TARGETS, - ), } -RG_TARGET_PLATFORM_PAIRS: list[tuple[str, str]] = [ - ("x86_64-unknown-linux-musl", "linux-x86_64"), - ("aarch64-unknown-linux-musl", "linux-aarch64"), - ("x86_64-apple-darwin", "macos-x86_64"), - ("aarch64-apple-darwin", "macos-aarch64"), - ("x86_64-pc-windows-msvc", "windows-x86_64"), - ("aarch64-pc-windows-msvc", "windows-aarch64"), -] -RG_TARGET_TO_PLATFORM = {target: platform for target, platform in RG_TARGET_PLATFORM_PAIRS} -DEFAULT_RG_TARGETS = [target for target, _ in RG_TARGET_PLATFORM_PAIRS] - -# urllib.request.urlopen() defaults to no timeout (can hang indefinitely), which is painful in CI. -DOWNLOAD_TIMEOUT_SECS = 60 - def _gha_enabled() -> bool: # GitHub Actions supports "workflow commands" (e.g. ::group:: / ::error::) that make logs @@ -141,22 +93,12 @@ def parse_args() -> argparse.Namespace: "--component", dest="components", action="append", - choices=tuple([CODEX_PACKAGE_COMPONENT, *BINARY_COMPONENTS, "rg"]), + choices=tuple([CODEX_PACKAGE_COMPONENT, *BINARY_COMPONENTS]), help=( "Limit installation to the specified components." " May be repeated. Defaults to codex-package and codex-responses-api-proxy." ), ) - parser.add_argument( - "--allow-legacy-codex-package", - action="store_true", - help=( - "Allow codex-package to be synthesized from legacy per-binary artifacts " - "when package archives are missing. Intended for CI compatibility only; " - "release staging should not use this. Automatically enabled for the " - "built-in default workflow." - ), - ) parser.add_argument( "root", nargs="?", @@ -179,7 +121,6 @@ def main() -> int: components = args.components or [CODEX_PACKAGE_COMPONENT, "codex-responses-api-proxy"] workflow_override = (args.workflow_url or "").strip() - use_default_workflow = not workflow_override workflow_url = workflow_override or DEFAULT_WORKFLOW_URL workflow_id = workflow_url.rstrip("/").split("/")[-1] @@ -190,28 +131,13 @@ def main() -> int: artifacts_dir = Path(artifacts_dir_str) _download_artifacts(workflow_id, artifacts_dir) if CODEX_PACKAGE_COMPONENT in components: - try: - install_codex_package_archives(artifacts_dir, vendor_dir, BINARY_TARGETS) - except FileNotFoundError: - if not (args.allow_legacy_codex_package or use_default_workflow): - raise - install_legacy_codex_package_layouts( - artifacts_dir, - vendor_dir, - BINARY_TARGETS, - manifest_path=RG_MANIFEST, - ) + install_codex_package_archives(artifacts_dir, vendor_dir, BINARY_TARGETS) install_binary_components( artifacts_dir, vendor_dir, [BINARY_COMPONENTS[name] for name in components if name in BINARY_COMPONENTS], ) - if "rg" in components: - with _gha_group("Fetch ripgrep binaries"): - print("Fetching ripgrep binaries...") - fetch_rg(vendor_dir, DEFAULT_RG_TARGETS, manifest_path=RG_MANIFEST) - print(f"Installed native dependencies into {vendor_dir}") return 0 @@ -263,156 +189,6 @@ def _install_single_codex_package_archive( return dest_dir -def install_legacy_codex_package_layouts( - artifacts_dir: Path, - vendor_dir: Path, - targets: Sequence[str], - *, - manifest_path: Path, -) -> None: - targets = list(targets) - print( - "Synthesizing Codex package layouts from legacy artifacts for targets: " - + ", ".join(targets) - ) - with tempfile.TemporaryDirectory(prefix="codex-legacy-package-") as legacy_vendor_dir_str: - legacy_vendor_dir = Path(legacy_vendor_dir_str) - install_binary_components( - artifacts_dir, - legacy_vendor_dir, - [ - BINARY_COMPONENTS["codex"], - BINARY_COMPONENTS["bwrap"], - BINARY_COMPONENTS["codex-windows-sandbox-setup"], - BINARY_COMPONENTS["codex-command-runner"], - ], - ) - fetch_rg(legacy_vendor_dir, targets, manifest_path=manifest_path) - - for target in targets: - dest_dir = vendor_dir / target - if dest_dir.exists(): - shutil.rmtree(dest_dir) - _build_legacy_codex_package_layout(legacy_vendor_dir / target, dest_dir, target) - print(f" synthesized {dest_dir}") - - -def _build_legacy_codex_package_layout( - legacy_target_dir: Path, - package_dir: Path, - target: str, -) -> None: - is_windows = "windows" in target - exe_suffix = ".exe" if is_windows else "" - package_dir.mkdir(parents=True) - - bin_dir = package_dir / "bin" - resources_dir = package_dir / "codex-resources" - path_dir = package_dir / "codex-path" - bin_dir.mkdir() - resources_dir.mkdir() - path_dir.mkdir() - - shutil.copy2( - legacy_target_dir / "codex" / f"codex{exe_suffix}", - bin_dir / f"codex{exe_suffix}", - ) - shutil.copy2( - legacy_target_dir / "path" / f"rg{exe_suffix}", - path_dir / f"rg{exe_suffix}", - ) - - if is_windows: - for helper in [ - "codex-command-runner.exe", - "codex-windows-sandbox-setup.exe", - ]: - shutil.copy2(legacy_target_dir / "codex" / helper, resources_dir / helper) - elif "linux" in target: - shutil.copy2(legacy_target_dir / "codex-resources" / "bwrap", resources_dir / "bwrap") - - write_json( - package_dir / "codex-package.json", - { - "layoutVersion": 1, - "version": "unknown", - "target": target, - "variant": "codex", - "entrypoint": f"bin/codex{exe_suffix}", - "resourcesDir": "codex-resources", - "pathDir": "codex-path", - }, - ) - - -def fetch_rg( - vendor_dir: Path, - targets: Sequence[str] | None = None, - *, - manifest_path: Path, -) -> list[Path]: - """Download ripgrep binaries described by the DotSlash manifest.""" - - if targets is None: - targets = DEFAULT_RG_TARGETS - - if not manifest_path.exists(): - raise FileNotFoundError(f"DotSlash manifest not found: {manifest_path}") - - manifest = _load_manifest(manifest_path) - platforms = manifest.get("platforms", {}) - - vendor_dir.mkdir(parents=True, exist_ok=True) - - targets = list(targets) - if not targets: - return [] - - task_configs: list[tuple[str, str, dict]] = [] - for target in targets: - platform_key = RG_TARGET_TO_PLATFORM.get(target) - if platform_key is None: - raise ValueError(f"Unsupported ripgrep target '{target}'.") - - platform_info = platforms.get(platform_key) - if platform_info is None: - raise RuntimeError(f"Platform '{platform_key}' not found in manifest {manifest_path}.") - - task_configs.append((target, platform_key, platform_info)) - - results: dict[str, Path] = {} - max_workers = min(len(task_configs), max(1, (os.cpu_count() or 1))) - - print("Installing ripgrep binaries for targets: " + ", ".join(targets)) - - with ThreadPoolExecutor(max_workers=max_workers) as executor: - future_map = { - executor.submit( - _fetch_single_rg, - vendor_dir, - target, - platform_key, - platform_info, - manifest_path, - ): target - for target, platform_key, platform_info in task_configs - } - - for future in as_completed(future_map): - target = future_map[future] - try: - results[target] = future.result() - except Exception as exc: - _gha_error( - title="ripgrep install failed", - message=f"target={target} error={exc!r}", - ) - raise RuntimeError(f"Failed to install ripgrep for target {target}.") from exc - print(f" installed ripgrep for {target}") - - return [results[target] for target in targets] - - def _download_artifacts(workflow_id: str, dest_dir: Path) -> None: cmd = [ "gh", @@ -436,7 +212,7 @@ def install_binary_components( return for component in selected_components: - component_targets = list(component.targets or BINARY_TARGETS) + component_targets = list(BINARY_TARGETS) print( f"Installing {component.binary_basename} binaries for targets: " @@ -466,7 +242,7 @@ def _install_single_binary( component: BinaryComponent, ) -> Path: artifact_subdir = artifact_dir_for_target(artifacts_dir, target) - archive_path = legacy_binary_archive_path(artifact_subdir, component.artifact_prefix, target) + archive_path = binary_archive_path(artifact_subdir, component.artifact_prefix, target) dest_dir = vendor_dir / target / component.dest_dir dest_dir.mkdir(parents=True, exist_ok=True) @@ -476,7 +252,7 @@ def _install_single_binary( ) dest = dest_dir / binary_name dest.unlink(missing_ok=True) - extract_archive(archive_path, "zst", None, dest) + extract_zstd_archive(archive_path, dest) if "windows" not in target: dest.chmod(0o755) return dest @@ -488,7 +264,7 @@ def _archive_name_for_target(artifact_prefix: str, target: str) -> str: return f"{artifact_prefix}-{target}.zst" -def legacy_binary_archive_path(artifact_dir: Path, artifact_prefix: str, target: str) -> Path: +def binary_archive_path(artifact_dir: Path, artifact_prefix: str, target: str) -> Path: archive_names = [_archive_name_for_target(artifact_prefix, target)] if artifact_dir.name == f"{target}-unsigned": archive_names.append(_archive_name_for_target(artifact_prefix, f"{target}-unsigned")) @@ -510,142 +286,12 @@ def artifact_dir_for_target(artifacts_dir: Path, target: str) -> Path: return artifacts_dir / target -def _fetch_single_rg( - vendor_dir: Path, - target: str, - platform_key: str, - platform_info: dict, - manifest_path: Path, -) -> Path: - providers = platform_info.get("providers", []) - if not providers: - raise RuntimeError(f"No providers listed for platform '{platform_key}' in {manifest_path}.") - - url = providers[0]["url"] - archive_format = platform_info.get("format", "zst") - archive_member = platform_info.get("path") - digest = platform_info.get("digest") - expected_size = platform_info.get("size") - - dest_dir = vendor_dir / target / "path" - dest_dir.mkdir(parents=True, exist_ok=True) - - is_windows = platform_key.startswith("win") - binary_name = "rg.exe" if is_windows else "rg" - dest = dest_dir / binary_name - - with tempfile.TemporaryDirectory() as tmp_dir_str: - tmp_dir = Path(tmp_dir_str) - archive_filename = os.path.basename(urlparse(url).path) - download_path = tmp_dir / archive_filename - print( - f" downloading ripgrep for {target} ({platform_key}) from {url}", - flush=True, - ) - try: - _download_file(url, download_path) - except Exception as exc: - _gha_error( - title="ripgrep download failed", - message=f"target={target} platform={platform_key} url={url} error={exc!r}", - ) - raise RuntimeError( - "Failed to download ripgrep " - f"(target={target}, platform={platform_key}, format={archive_format}, " - f"expected_size={expected_size!r}, digest={digest!r}, url={url}, dest={download_path})." - ) from exc - - dest.unlink(missing_ok=True) - try: - extract_archive(download_path, archive_format, archive_member, dest) - except Exception as exc: - raise RuntimeError( - "Failed to extract ripgrep " - f"(target={target}, platform={platform_key}, format={archive_format}, " - f"member={archive_member!r}, url={url}, archive={download_path})." - ) from exc - - if not is_windows: - dest.chmod(0o755) - - return dest - - -def _download_file(url: str, dest: Path) -> None: - dest.parent.mkdir(parents=True, exist_ok=True) - dest.unlink(missing_ok=True) - - with urlopen(url, timeout=DOWNLOAD_TIMEOUT_SECS) as response, open(dest, "wb") as out: - shutil.copyfileobj(response, out) - - -def extract_archive( - archive_path: Path, - archive_format: str, - archive_member: str | None, - dest: Path, -) -> None: +def extract_zstd_archive(archive_path: Path, dest: Path) -> None: dest.parent.mkdir(parents=True, exist_ok=True) - if archive_format == "zst": - output_path = archive_path.parent / dest.name - subprocess.check_call( - ["zstd", "-f", "-d", str(archive_path), "-o", str(output_path)] - ) - shutil.move(str(output_path), dest) - return - - if archive_format == "tar.gz": - if not archive_member: - raise RuntimeError("Missing 'path' for tar.gz archive in DotSlash manifest.") - with tarfile.open(archive_path, "r:gz") as tar: - try: - member = tar.getmember(archive_member) - except KeyError as exc: - raise RuntimeError( - f"Entry '{archive_member}' not found in archive {archive_path}." - ) from exc - tar.extract(member, path=archive_path.parent, filter="data") - extracted = archive_path.parent / archive_member - shutil.move(str(extracted), dest) - return - - if archive_format == "zip": - if not archive_member: - raise RuntimeError("Missing 'path' for zip archive in DotSlash manifest.") - with zipfile.ZipFile(archive_path) as archive: - try: - with archive.open(archive_member) as src, open(dest, "wb") as out: - shutil.copyfileobj(src, out) - except KeyError as exc: - raise RuntimeError( - f"Entry '{archive_member}' not found in archive {archive_path}." - ) from exc - return - - raise RuntimeError(f"Unsupported archive format '{archive_format}'.") - - -def _load_manifest(manifest_path: Path) -> dict: - cmd = ["dotslash", "--", "parse", str(manifest_path)] - stdout = subprocess.check_output(cmd, text=True) - try: - manifest = json.loads(stdout) - except json.JSONDecodeError as exc: - raise RuntimeError(f"Invalid DotSlash manifest output from {manifest_path}.") from exc - - if not isinstance(manifest, dict): - raise RuntimeError( - f"Unexpected DotSlash manifest structure for {manifest_path}: {type(manifest)!r}" - ) - - return manifest - - -def write_json(path: Path, value: object) -> None: - with open(path, "w", encoding="utf-8") as out: - json.dump(value, out, indent=2) - out.write("\n") + output_path = archive_path.parent / dest.name + subprocess.check_call(["zstd", "-f", "-d", str(archive_path), "-o", str(output_path)]) + shutil.move(str(output_path), dest) if __name__ == "__main__": diff --git a/codex-cli/scripts/test_install_native_deps.py b/codex-cli/scripts/test_install_native_deps.py new file mode 100644 index 00000000000..d2131502fe6 --- /dev/null +++ b/codex-cli/scripts/test_install_native_deps.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 + +from contextlib import redirect_stdout +import importlib.util +import io +from pathlib import Path +import tarfile +import tempfile +import unittest + + +INSTALL_SCRIPT = Path(__file__).resolve().parent / "install_native_deps.py" +SPEC = importlib.util.spec_from_file_location("install_native_deps", INSTALL_SCRIPT) +if SPEC is None or SPEC.loader is None: + raise RuntimeError(f"Unable to load module from {INSTALL_SCRIPT}") + +install_native_deps = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(install_native_deps) + + +class InstallCodexPackageArchivesTest(unittest.TestCase): + def test_installs_codex_package_archive(self) -> None: + target = "x86_64-unknown-linux-musl" + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + artifact_dir = root / "artifacts" / target + package_src = root / "package-src" + vendor_dir = root / "vendor" + artifact_dir.mkdir(parents=True) + (package_src / "bin").mkdir(parents=True) + (package_src / "bin" / "codex").write_text("codex\n", encoding="utf-8") + (package_src / "codex-package.json").write_text("{}\n", encoding="utf-8") + + archive_path = artifact_dir / f"codex-package-{target}.tar.gz" + with tarfile.open(archive_path, "w:gz") as archive: + archive.add(package_src / "bin", arcname="bin") + archive.add(package_src / "codex-package.json", arcname="codex-package.json") + + with redirect_stdout(io.StringIO()): + install_native_deps.install_codex_package_archives( + root / "artifacts", + vendor_dir, + [target], + ) + + self.assertEqual( + sorted( + path.relative_to(vendor_dir / target) + for path in (vendor_dir / target).rglob("*") + ), + [ + Path("bin"), + Path("bin/codex"), + Path("codex-package.json"), + ], + ) + + def test_missing_codex_package_archive_errors(self) -> None: + target = "x86_64-unknown-linux-musl" + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + + with redirect_stdout(io.StringIO()): + with self.assertRaisesRegex( + FileNotFoundError, + "Expected package archive not found", + ): + install_native_deps.install_codex_package_archives( + root / "artifacts", + root / "vendor", + [target], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/stage_npm_packages.py b/scripts/stage_npm_packages.py index 4eb69053ebc..79120ea2ebf 100755 --- a/scripts/stage_npm_packages.py +++ b/scripts/stage_npm_packages.py @@ -66,15 +66,6 @@ def parse_args() -> argparse.Namespace: "Intended for CI compatibility only; release staging should not use this." ), ) - parser.add_argument( - "--allow-legacy-codex-package", - action="store_true", - help=( - "Allow codex-package layouts to be synthesized from legacy per-binary " - "workflow artifacts. Intended for CI compatibility only; release staging " - "should not use this." - ), - ) return parser.parse_args() @@ -131,15 +122,11 @@ def install_native_components( workflow_url: str, components: set[str], vendor_root: Path, - *, - allow_legacy_codex_package: bool, ) -> None: if not components: return cmd = [str(INSTALL_NATIVE_DEPS), "--workflow-url", workflow_url] - if allow_legacy_codex_package: - cmd.append("--allow-legacy-codex-package") for component in sorted(components): cmd.extend(["--component", component]) cmd.append(str(vendor_root)) @@ -187,7 +174,6 @@ def main() -> int: workflow_url, native_components_to_install, vendor_temp_root, - allow_legacy_codex_package=args.allow_legacy_codex_package, ) vendor_src = vendor_temp_root / "vendor" From 9ccb02339c06bb35f0a18f9053f9f0982b16309f Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 21 May 2026 11:52:57 -0700 Subject: [PATCH 2/7] npm: drop unrun install_native_deps test --- codex-cli/scripts/test_install_native_deps.py | 76 ------------------- 1 file changed, 76 deletions(-) delete mode 100644 codex-cli/scripts/test_install_native_deps.py diff --git a/codex-cli/scripts/test_install_native_deps.py b/codex-cli/scripts/test_install_native_deps.py deleted file mode 100644 index d2131502fe6..00000000000 --- a/codex-cli/scripts/test_install_native_deps.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python3 - -from contextlib import redirect_stdout -import importlib.util -import io -from pathlib import Path -import tarfile -import tempfile -import unittest - - -INSTALL_SCRIPT = Path(__file__).resolve().parent / "install_native_deps.py" -SPEC = importlib.util.spec_from_file_location("install_native_deps", INSTALL_SCRIPT) -if SPEC is None or SPEC.loader is None: - raise RuntimeError(f"Unable to load module from {INSTALL_SCRIPT}") - -install_native_deps = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(install_native_deps) - - -class InstallCodexPackageArchivesTest(unittest.TestCase): - def test_installs_codex_package_archive(self) -> None: - target = "x86_64-unknown-linux-musl" - with tempfile.TemporaryDirectory() as temp_dir: - root = Path(temp_dir) - artifact_dir = root / "artifacts" / target - package_src = root / "package-src" - vendor_dir = root / "vendor" - artifact_dir.mkdir(parents=True) - (package_src / "bin").mkdir(parents=True) - (package_src / "bin" / "codex").write_text("codex\n", encoding="utf-8") - (package_src / "codex-package.json").write_text("{}\n", encoding="utf-8") - - archive_path = artifact_dir / f"codex-package-{target}.tar.gz" - with tarfile.open(archive_path, "w:gz") as archive: - archive.add(package_src / "bin", arcname="bin") - archive.add(package_src / "codex-package.json", arcname="codex-package.json") - - with redirect_stdout(io.StringIO()): - install_native_deps.install_codex_package_archives( - root / "artifacts", - vendor_dir, - [target], - ) - - self.assertEqual( - sorted( - path.relative_to(vendor_dir / target) - for path in (vendor_dir / target).rglob("*") - ), - [ - Path("bin"), - Path("bin/codex"), - Path("codex-package.json"), - ], - ) - - def test_missing_codex_package_archive_errors(self) -> None: - target = "x86_64-unknown-linux-musl" - with tempfile.TemporaryDirectory() as temp_dir: - root = Path(temp_dir) - - with redirect_stdout(io.StringIO()): - with self.assertRaisesRegex( - FileNotFoundError, - "Expected package archive not found", - ): - install_native_deps.install_codex_package_archives( - root / "artifacts", - root / "vendor", - [target], - ) - - -if __name__ == "__main__": - unittest.main() From 64da9ab3e56f6105b714de8176a672947f946aa6 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 21 May 2026 11:57:59 -0700 Subject: [PATCH 3/7] npm: remove unused missing-native flag --- codex-cli/scripts/build_npm_package.py | 15 --------------- scripts/stage_npm_packages.py | 19 ++----------------- 2 files changed, 2 insertions(+), 32 deletions(-) diff --git a/codex-cli/scripts/build_npm_package.py b/codex-cli/scripts/build_npm_package.py index 261b9e0b614..2b6a6de4d0b 100755 --- a/codex-cli/scripts/build_npm_package.py +++ b/codex-cli/scripts/build_npm_package.py @@ -140,16 +140,6 @@ def parse_args() -> argparse.Namespace: type=Path, help="Directory containing pre-installed native binaries to bundle (vendor root).", ) - parser.add_argument( - "--allow-missing-native-component", - dest="allow_missing_native_components", - action="append", - default=[], - help=( - "Native component that may be absent from --vendor-src. Intended for CI " - "compatibility with older artifact workflows; releases should not use this." - ), - ) return parser.parse_args() @@ -190,7 +180,6 @@ def main() -> int: staging_dir, native_components, target_filter={target_filter} if target_filter else None, - allow_missing_components=set(args.allow_missing_native_components), ) if release_version: @@ -376,7 +365,6 @@ def copy_native_binaries( staging_dir: Path, components: list[str], target_filter: set[str] | None = None, - allow_missing_components: set[str] | None = None, ) -> None: vendor_src = vendor_src.resolve() if not vendor_src.exists(): @@ -387,7 +375,6 @@ def copy_native_binaries( for component in components if component == CODEX_PACKAGE_COMPONENT or component in COMPONENT_DEST_DIR } - allow_missing_components = allow_missing_components or set() if not components_set: return @@ -431,8 +418,6 @@ def copy_native_binaries( src_component_dir = target_dir / dest_dir_name if not src_component_dir.exists(): - if component in allow_missing_components: - continue raise RuntimeError( f"Missing native component '{component}' in vendor source: {src_component_dir}" ) diff --git a/scripts/stage_npm_packages.py b/scripts/stage_npm_packages.py index 79120ea2ebf..8c189496131 100755 --- a/scripts/stage_npm_packages.py +++ b/scripts/stage_npm_packages.py @@ -56,16 +56,6 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Retain temporary staging directories instead of deleting them.", ) - parser.add_argument( - "--allow-missing-native-component", - dest="allow_missing_native_components", - action="append", - default=[], - help=( - "Native component that may be absent from reused workflow artifacts. " - "Intended for CI compatibility only; release staging should not use this." - ), - ) return parser.parse_args() @@ -155,8 +145,6 @@ def main() -> int: packages = expand_packages(list(args.packages)) native_components = collect_native_components(packages) - allow_missing_native_components = set(args.allow_missing_native_components) - native_components_to_install = native_components - allow_missing_native_components vendor_temp_root: Path | None = None vendor_src: Path | None = None @@ -165,14 +153,14 @@ def main() -> int: final_messages = [] try: - if native_components_to_install: + if native_components: workflow_url, resolved_head_sha = resolve_workflow_url( args.release_version, args.workflow_url ) vendor_temp_root = Path(tempfile.mkdtemp(prefix="npm-native-", dir=runner_temp)) install_native_components( workflow_url, - native_components_to_install, + native_components, vendor_temp_root, ) vendor_src = vendor_temp_root / "vendor" @@ -199,9 +187,6 @@ def main() -> int: if vendor_src is not None: cmd.extend(["--vendor-src", str(vendor_src)]) - for component in sorted(allow_missing_native_components): - cmd.extend(["--allow-missing-native-component", component]) - try: run_command(cmd) finally: From e4284c0a422f77259eddd5072720a1ef4ea0f6cd Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 21 May 2026 12:00:55 -0700 Subject: [PATCH 4/7] npm: copy full codex package payload --- codex-cli/scripts/build_npm_package.py | 41 +------------------------- 1 file changed, 1 insertion(+), 40 deletions(-) diff --git a/codex-cli/scripts/build_npm_package.py b/codex-cli/scripts/build_npm_package.py index 2b6a6de4d0b..39756e00e0f 100755 --- a/codex-cli/scripts/build_npm_package.py +++ b/codex-cli/scripts/build_npm_package.py @@ -16,7 +16,6 @@ CODEX_SDK_ROOT = REPO_ROOT / "sdk" / "typescript" CODEX_NPM_NAME = "@openai/codex" CODEX_PACKAGE_COMPONENT = "codex-package" -CODEX_PACKAGE_ENTRIES = ("codex-package.json", "bin", "codex-resources", "codex-path") # `npm_name` is the local optional-dependency alias consumed by `bin/codex.js`. # The underlying package published to npm is always `@openai/codex`. @@ -397,17 +396,9 @@ def copy_native_binaries( dest_target_dir = vendor_dest / target_dir.name if CODEX_PACKAGE_COMPONENT in components_set: - validate_codex_package_dir(target_dir) if dest_target_dir.exists(): shutil.rmtree(dest_target_dir) - dest_target_dir.mkdir(parents=True, exist_ok=True) - for entry in CODEX_PACKAGE_ENTRIES: - src = target_dir / entry - dest = dest_target_dir / entry - if src.is_dir(): - shutil.copytree(src, dest) - else: - shutil.copy2(src, dest) + shutil.copytree(target_dir, dest_target_dir) else: dest_target_dir.mkdir(parents=True, exist_ok=True) @@ -433,36 +424,6 @@ def copy_native_binaries( missing_list = ", ".join(missing_targets) raise RuntimeError(f"Missing target directories in vendor source: {missing_list}") - -def validate_codex_package_dir(package_dir: Path) -> None: - is_windows = "windows" in package_dir.name - required_files = [ - Path("codex-package.json"), - Path("bin") / ("codex.exe" if is_windows else "codex"), - Path("codex-path") / ("rg.exe" if is_windows else "rg"), - ] - - if "linux" in package_dir.name: - required_files.append(Path("codex-resources") / "bwrap") - - if is_windows: - required_files.extend( - [ - Path("codex-resources") / "codex-command-runner.exe", - Path("codex-resources") / "codex-windows-sandbox-setup.exe", - ] - ) - - missing_files = [ - str(relative_path) - for relative_path in required_files - if not (package_dir / relative_path).is_file() - ] - if missing_files: - missing = ", ".join(missing_files) - raise RuntimeError(f"Missing files in Codex package directory {package_dir}: {missing}") - - def run_npm_pack(staging_dir: Path, output_path: Path) -> Path: output_path = output_path.resolve() output_path.parent.mkdir(parents=True, exist_ok=True) From df7638b6fe7c80ac0412f00fb856e5ee8cd761bc Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 21 May 2026 12:08:32 -0700 Subject: [PATCH 5/7] npm: remove native component path map --- codex-cli/scripts/README.md | 4 +-- codex-cli/scripts/build_npm_package.py | 26 +++----------- scripts/stage_npm_packages.py | 47 ++++++++++++++++---------- 3 files changed, 36 insertions(+), 41 deletions(-) diff --git a/codex-cli/scripts/README.md b/codex-cli/scripts/README.md index ce58097db54..db81ceff4e4 100644 --- a/codex-cli/scripts/README.md +++ b/codex-cli/scripts/README.md @@ -11,8 +11,8 @@ example, to stage the CLI, responses proxy, and SDK packages for version `0.6.0` --package codex-sdk ``` -This downloads the native package archive artifacts once, hydrates `vendor/` for each -package, and writes tarballs to `dist/npm/`. +This downloads the required native package archive artifacts, hydrates `vendor/` for +each package, and writes tarballs to `dist/npm/`. When `--package codex` is provided, the staging helper builds the lightweight `@openai/codex` meta package plus all platform-native `@openai/codex` variants diff --git a/codex-cli/scripts/build_npm_package.py b/codex-cli/scripts/build_npm_package.py index 39756e00e0f..9ef837d8941 100755 --- a/codex-cli/scripts/build_npm_package.py +++ b/codex-cli/scripts/build_npm_package.py @@ -87,16 +87,6 @@ PACKAGE_CHOICES = tuple(PACKAGE_NATIVE_COMPONENTS) -COMPONENT_DEST_DIR: dict[str, str] = { - "bwrap": "codex-resources", - "codex": "codex", - "codex-responses-api-proxy": "codex-responses-api-proxy", - "codex-windows-sandbox-setup": "codex", - "codex-command-runner": "codex", - "rg": "path", -} - - def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Build or stage the Codex CLI npm package.") parser.add_argument( @@ -369,11 +359,7 @@ def copy_native_binaries( if not vendor_src.exists(): raise RuntimeError(f"Vendor source directory not found: {vendor_src}") - components_set = { - component - for component in components - if component == CODEX_PACKAGE_COMPONENT or component in COMPONENT_DEST_DIR - } + components_set = set(components) if not components_set: return @@ -402,18 +388,14 @@ def copy_native_binaries( else: dest_target_dir.mkdir(parents=True, exist_ok=True) - for component in components_set - {CODEX_PACKAGE_COMPONENT}: - dest_dir_name = COMPONENT_DEST_DIR.get(component) - if dest_dir_name is None: - continue - - src_component_dir = target_dir / dest_dir_name + for component in sorted(components_set - {CODEX_PACKAGE_COMPONENT}): + src_component_dir = target_dir / component if not src_component_dir.exists(): raise RuntimeError( f"Missing native component '{component}' in vendor source: {src_component_dir}" ) - dest_component_dir = dest_target_dir / dest_dir_name + dest_component_dir = dest_target_dir / component if dest_component_dir.exists(): shutil.rmtree(dest_component_dir) shutil.copytree(src_component_dir, dest_component_dir) diff --git a/scripts/stage_npm_packages.py b/scripts/stage_npm_packages.py index 8c189496131..e97ae6a882f 100755 --- a/scripts/stage_npm_packages.py +++ b/scripts/stage_npm_packages.py @@ -59,11 +59,20 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() -def collect_native_components(packages: list[str]) -> set[str]: - components: set[str] = set() +def native_components_for_package(package: str) -> tuple[str, ...]: + return tuple(sorted(PACKAGE_NATIVE_COMPONENTS.get(package, []))) + + +def collect_native_component_sets(packages: list[str]) -> list[tuple[str, ...]]: + component_sets: list[tuple[str, ...]] = [] + seen: set[tuple[str, ...]] = set() for package in packages: - components.update(PACKAGE_NATIVE_COMPONENTS.get(package, [])) - return components + components = native_components_for_package(package) + if not components or components in seen: + continue + seen.add(components) + component_sets.append(components) + return component_sets def expand_packages(packages: list[str]) -> list[str]: @@ -144,26 +153,28 @@ def main() -> int: runner_temp = Path(os.environ.get("RUNNER_TEMP", tempfile.gettempdir())) packages = expand_packages(list(args.packages)) - native_components = collect_native_components(packages) + native_component_sets = collect_native_component_sets(packages) - vendor_temp_root: Path | None = None - vendor_src: Path | None = None + vendor_temp_roots: list[Path] = [] + vendor_src_by_components: dict[tuple[str, ...], Path] = {} resolved_head_sha: str | None = None final_messages = [] try: - if native_components: + if native_component_sets: workflow_url, resolved_head_sha = resolve_workflow_url( args.release_version, args.workflow_url ) - vendor_temp_root = Path(tempfile.mkdtemp(prefix="npm-native-", dir=runner_temp)) - install_native_components( - workflow_url, - native_components, - vendor_temp_root, - ) - vendor_src = vendor_temp_root / "vendor" + for components in native_component_sets: + vendor_temp_root = Path(tempfile.mkdtemp(prefix="npm-native-", dir=runner_temp)) + vendor_temp_roots.append(vendor_temp_root) + install_native_components( + workflow_url, + set(components), + vendor_temp_root, + ) + vendor_src_by_components[components] = vendor_temp_root / "vendor" if resolved_head_sha: print(f"should `git checkout {resolved_head_sha}`") @@ -184,6 +195,7 @@ def main() -> int: str(pack_output), ] + vendor_src = vendor_src_by_components.get(native_components_for_package(package)) if vendor_src is not None: cmd.extend(["--vendor-src", str(vendor_src)]) @@ -195,8 +207,9 @@ def main() -> int: final_messages.append(f"Staged {package} at {pack_output}") finally: - if vendor_temp_root is not None and not args.keep_staging_dirs: - shutil.rmtree(vendor_temp_root, ignore_errors=True) + if not args.keep_staging_dirs: + for vendor_temp_root in vendor_temp_roots: + shutil.rmtree(vendor_temp_root, ignore_errors=True) for msg in final_messages: print(msg) From 89ce55e58f1df76f02b5b5ac7b86d06569fad506 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 21 May 2026 12:51:47 -0700 Subject: [PATCH 6/7] npm: cache workflow artifacts while staging packages --- codex-cli/scripts/build_npm_package.py | 15 +- codex-cli/scripts/install_native_deps.py | 176 +++++++++++++++++++---- scripts/stage_npm_packages.py | 40 +++++- 3 files changed, 197 insertions(+), 34 deletions(-) diff --git a/codex-cli/scripts/build_npm_package.py b/codex-cli/scripts/build_npm_package.py index 9ef837d8941..663872c11e3 100755 --- a/codex-cli/scripts/build_npm_package.py +++ b/codex-cli/scripts/build_npm_package.py @@ -3,6 +3,7 @@ import argparse import json +import os import shutil import subprocess import sys @@ -324,8 +325,10 @@ def compute_platform_package_version(version: str, platform_tag: str) -> str: def run_command(cmd: list[str], cwd: Path | None = None) -> None: - print("+", " ".join(cmd)) - subprocess.run(cmd, cwd=cwd, check=True) + print("+", " ".join(cmd), flush=True) + env = os.environ.copy() + env.setdefault("CI", "true") + subprocess.run(cmd, cwd=cwd, env=env, check=True) def stage_codex_sdk_sources(staging_dir: Path) -> None: @@ -412,9 +415,17 @@ def run_npm_pack(staging_dir: Path, output_path: Path) -> Path: with tempfile.TemporaryDirectory(prefix="codex-npm-pack-") as pack_dir_str: pack_dir = Path(pack_dir_str) + npm_cache_dir = pack_dir / "npm-cache" + npm_logs_dir = pack_dir / "npm-logs" + npm_cache_dir.mkdir() + npm_logs_dir.mkdir() + env = os.environ.copy() + env["NPM_CONFIG_CACHE"] = str(npm_cache_dir) + env["NPM_CONFIG_LOGS_DIR"] = str(npm_logs_dir) stdout = subprocess.check_output( ["npm", "pack", "--json", "--pack-destination", str(pack_dir)], cwd=staging_dir, + env=env, text=True, ) try: diff --git a/codex-cli/scripts/install_native_deps.py b/codex-cli/scripts/install_native_deps.py index cd6c9063457..27ffc5d8408 100755 --- a/codex-cli/scripts/install_native_deps.py +++ b/codex-cli/scripts/install_native_deps.py @@ -16,6 +16,7 @@ SCRIPT_DIR = Path(__file__).resolve().parent CODEX_CLI_ROOT = SCRIPT_DIR.parent DEFAULT_WORKFLOW_URL = "https://github.com/openai/codex/actions/runs/26201494185" # rust-v0.133.0-alpha.4 +GITHUB_REPO = "openai/codex" VENDOR_DIR_NAME = "vendor" BINARY_TARGETS = ( "x86_64-unknown-linux-musl", @@ -35,6 +36,12 @@ class BinaryComponent: binary_basename: str # executable name inside dest_dir (before optional .exe) +@dataclass(frozen=True) +class WorkflowArtifact: + name: str + size_in_bytes: int + + BINARY_COMPONENTS = { "codex-responses-api-proxy": BinaryComponent( artifact_prefix="codex-responses-api-proxy", @@ -99,6 +106,14 @@ def parse_args() -> argparse.Namespace: " May be repeated. Defaults to codex-package and codex-responses-api-proxy." ), ) + parser.add_argument( + "--artifacts-dir", + type=Path, + help=( + "Directory used to cache downloaded workflow artifacts. Defaults to a " + "temporary directory." + ), + ) parser.add_argument( "root", nargs="?", @@ -124,24 +139,91 @@ def main() -> int: workflow_url = workflow_override or DEFAULT_WORKFLOW_URL workflow_id = workflow_url.rstrip("/").split("/")[-1] - print(f"Downloading native artifacts from workflow {workflow_id}...") + print(f"Downloading native artifacts from workflow {workflow_id}...", flush=True) with _gha_group(f"Download native artifacts from workflow {workflow_id}"): - with tempfile.TemporaryDirectory(prefix="codex-native-artifacts-") as artifacts_dir_str: - artifacts_dir = Path(artifacts_dir_str) - _download_artifacts(workflow_id, artifacts_dir) - if CODEX_PACKAGE_COMPONENT in components: - install_codex_package_archives(artifacts_dir, vendor_dir, BINARY_TARGETS) - install_binary_components( - artifacts_dir, - vendor_dir, - [BINARY_COMPONENTS[name] for name in components if name in BINARY_COMPONENTS], - ) + if args.artifacts_dir is not None: + artifacts_dir = args.artifacts_dir.resolve() + artifacts_dir.mkdir(parents=True, exist_ok=True) + install_from_workflow_artifacts(workflow_id, artifacts_dir, components, vendor_dir) + else: + with tempfile.TemporaryDirectory(prefix="codex-native-artifacts-") as artifacts_dir_str: + artifacts_dir = Path(artifacts_dir_str) + install_from_workflow_artifacts( + workflow_id, + artifacts_dir, + components, + vendor_dir, + ) - print(f"Installed native dependencies into {vendor_dir}") + print(f"Installed native dependencies into {vendor_dir}", flush=True) return 0 +def install_from_workflow_artifacts( + workflow_id: str, + artifacts_dir: Path, + components: Sequence[str], + vendor_dir: Path, +) -> None: + artifact_names = select_target_artifacts(workflow_id, components) + _download_artifacts(workflow_id, artifacts_dir, artifact_names) + if CODEX_PACKAGE_COMPONENT in components: + install_codex_package_archives(artifacts_dir, vendor_dir, BINARY_TARGETS) + install_binary_components( + artifacts_dir, + vendor_dir, + [BINARY_COMPONENTS[name] for name in components if name in BINARY_COMPONENTS], + ) + + +def select_target_artifacts( + workflow_id: str, + components: Sequence[str], +) -> list[WorkflowArtifact]: + needs_target_artifacts = CODEX_PACKAGE_COMPONENT in components or any( + component in BINARY_COMPONENTS for component in components + ) + if not needs_target_artifacts: + return [] + + artifacts_by_name = { + artifact.name: artifact for artifact in list_workflow_artifacts(workflow_id) + } + selected_artifacts: list[WorkflowArtifact] = [] + for target in BINARY_TARGETS: + for artifact_name in [target, f"{target}-unsigned"]: + artifact = artifacts_by_name.get(artifact_name) + if artifact is not None: + selected_artifacts.append(artifact) + break + else: + raise FileNotFoundError( + f"Expected workflow artifact not found for target {target}" + ) + + return selected_artifacts + + +def list_workflow_artifacts(workflow_id: str) -> list[WorkflowArtifact]: + stdout = subprocess.check_output( + [ + "gh", + "api", + f"repos/{GITHUB_REPO}/actions/runs/{workflow_id}/artifacts", + "--paginate", + "--jq", + ".artifacts[] | [.name, .size_in_bytes] | @tsv", + ], + text=True, + ) + artifacts: list[WorkflowArtifact] = [] + for line in stdout.splitlines(): + name, size_in_bytes = line.split("\t", 1) + artifacts.append(WorkflowArtifact(name=name, size_in_bytes=int(size_in_bytes))) + return artifacts + + def install_codex_package_archives( artifacts_dir: Path, vendor_dir: Path, @@ -151,7 +233,10 @@ def install_codex_package_archives( if not targets: return - print("Installing Codex package archives for targets: " + ", ".join(targets)) + print( + "Installing Codex package archives for targets: " + ", ".join(targets), + flush=True, + ) max_workers = min(len(targets), max(1, (os.cpu_count() or 1))) with ThreadPoolExecutor(max_workers=max_workers) as executor: futures = { @@ -165,7 +250,7 @@ def install_codex_package_archives( } for future in as_completed(futures): installed_path = future.result() - print(f" installed {installed_path}") + print(f" installed {installed_path}", flush=True) def _install_single_codex_package_archive( @@ -189,18 +274,43 @@ def _install_single_codex_package_archive( return dest_dir -def _download_artifacts(workflow_id: str, dest_dir: Path) -> None: - cmd = [ - "gh", - "run", - "download", - "--dir", - str(dest_dir), - "--repo", - "openai/codex", - workflow_id, - ] - subprocess.check_call(cmd) +def _download_artifacts( + workflow_id: str, + dest_dir: Path, + artifacts: Sequence[WorkflowArtifact], +) -> None: + total_bytes = sum(artifact.size_in_bytes for artifact in artifacts) + print( + f"Downloading {len(artifacts)} artifacts ({format_bytes(total_bytes)})", + flush=True, + ) + for artifact in artifacts: + artifact_dir = dest_dir / artifact.name + if artifact_dir.is_dir() and any(artifact_dir.iterdir()): + print( + f" using cached {artifact.name} ({format_bytes(artifact.size_in_bytes)})", + flush=True, + ) + continue + + artifact_dir.mkdir(parents=True, exist_ok=True) + print( + f" downloading {artifact.name} ({format_bytes(artifact.size_in_bytes)})", + flush=True, + ) + cmd = [ + "gh", + "run", + "download", + "--name", + artifact.name, + "--dir", + str(artifact_dir), + "--repo", + GITHUB_REPO, + workflow_id, + ] + subprocess.check_call(cmd) def install_binary_components( @@ -216,7 +326,8 @@ def install_binary_components( print( f"Installing {component.binary_basename} binaries for targets: " - + ", ".join(component_targets) + + ", ".join(component_targets), + flush=True, ) max_workers = min(len(component_targets), max(1, (os.cpu_count() or 1))) with ThreadPoolExecutor(max_workers=max_workers) as executor: @@ -232,7 +343,16 @@ def install_binary_components( } for future in as_completed(futures): installed_path = future.result() - print(f" installed {installed_path}") + print(f" installed {installed_path}", flush=True) + + +def format_bytes(size_in_bytes: int) -> str: + value = float(size_in_bytes) + for unit in ["B", "KiB", "MiB"]: + if value < 1024: + return f"{value:.1f} {unit}" + value /= 1024 + return f"{value:.1f} GiB" def _install_single_binary( diff --git a/scripts/stage_npm_packages.py b/scripts/stage_npm_packages.py index e97ae6a882f..3af9cfe6fba 100755 --- a/scripts/stage_npm_packages.py +++ b/scripts/stage_npm_packages.py @@ -121,11 +121,18 @@ def install_native_components( workflow_url: str, components: set[str], vendor_root: Path, + artifacts_dir: Path, ) -> None: if not components: return - cmd = [str(INSTALL_NATIVE_DEPS), "--workflow-url", workflow_url] + cmd = [ + str(INSTALL_NATIVE_DEPS), + "--workflow-url", + workflow_url, + "--artifacts-dir", + str(artifacts_dir), + ] for component in sorted(components): cmd.extend(["--component", component]) cmd.append(str(vendor_root)) @@ -133,7 +140,7 @@ def install_native_components( def run_command(cmd: list[str]) -> None: - print("+", " ".join(cmd)) + print("+", " ".join(cmd), flush=True) subprocess.run(cmd, cwd=REPO_ROOT, check=True) @@ -154,9 +161,19 @@ def main() -> int: packages = expand_packages(list(args.packages)) native_component_sets = collect_native_component_sets(packages) + print("Expanded packages: " + ", ".join(packages), flush=True) + if native_component_sets: + component_sets = [ + "(" + ", ".join(components) + ")" for components in native_component_sets + ] + print( + "Native component sets: " + ", ".join(component_sets), + flush=True, + ) vendor_temp_roots: list[Path] = [] vendor_src_by_components: dict[tuple[str, ...], Path] = {} + artifacts_temp_root: Path | None = None resolved_head_sha: str | None = None final_messages = [] @@ -166,22 +183,35 @@ def main() -> int: workflow_url, resolved_head_sha = resolve_workflow_url( args.release_version, args.workflow_url ) + print(f"Using native artifacts from {workflow_url}", flush=True) + artifacts_temp_root = Path( + tempfile.mkdtemp(prefix="npm-native-artifacts-", dir=runner_temp) + ) + print(f"Caching downloaded artifacts in {artifacts_temp_root}", flush=True) for components in native_component_sets: vendor_temp_root = Path(tempfile.mkdtemp(prefix="npm-native-", dir=runner_temp)) vendor_temp_roots.append(vendor_temp_root) + print( + "Installing native components " + + ", ".join(components) + + f" into {vendor_temp_root}", + flush=True, + ) install_native_components( workflow_url, set(components), vendor_temp_root, + artifacts_temp_root, ) vendor_src_by_components[components] = vendor_temp_root / "vendor" if resolved_head_sha: - print(f"should `git checkout {resolved_head_sha}`") + print(f"should `git checkout {resolved_head_sha}`", flush=True) for package in packages: staging_dir = Path(tempfile.mkdtemp(prefix=f"npm-stage-{package}-", dir=runner_temp)) pack_output = output_dir / tarball_name_for_package(package, args.release_version) + print(f"Staging {package} in {staging_dir}", flush=True) cmd = [ str(BUILD_SCRIPT), @@ -210,9 +240,11 @@ def main() -> int: if not args.keep_staging_dirs: for vendor_temp_root in vendor_temp_roots: shutil.rmtree(vendor_temp_root, ignore_errors=True) + if artifacts_temp_root is not None: + shutil.rmtree(artifacts_temp_root, ignore_errors=True) for msg in final_messages: - print(msg) + print(msg, flush=True) return 0 From 45308f27660d5c4adb3bf88ad9fe28fff1b886d0 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 21 May 2026 13:33:27 -0700 Subject: [PATCH 7/7] npm: inline native staging helper --- codex-cli/scripts/README.md | 6 +- codex-cli/scripts/build_npm_package.py | 4 +- codex-cli/scripts/install_native_deps.py | 420 ----------------------- scripts/stage_npm_packages.py | 332 +++++++++++++++++- 4 files changed, 324 insertions(+), 438 deletions(-) delete mode 100755 codex-cli/scripts/install_native_deps.py diff --git a/codex-cli/scripts/README.md b/codex-cli/scripts/README.md index db81ceff4e4..65923151a0c 100644 --- a/codex-cli/scripts/README.md +++ b/codex-cli/scripts/README.md @@ -18,6 +18,6 @@ When `--package codex` is provided, the staging helper builds the lightweight `@openai/codex` meta package plus all platform-native `@openai/codex` variants that are later published under platform-specific dist-tags. -If you need to invoke `build_npm_package.py` directly, run -`codex-cli/scripts/install_native_deps.py --component codex-package` first and pass -`--vendor-src` pointing to the directory that contains the populated `vendor/` tree. +Direct `build_npm_package.py` invocations are still useful for package-specific +debugging, but native packages expect `--vendor-src` to point at a prehydrated +`vendor/` tree. Release packaging should use `scripts/stage_npm_packages.py`. diff --git a/codex-cli/scripts/build_npm_package.py b/codex-cli/scripts/build_npm_package.py index 663872c11e3..60f6ca7a9d5 100755 --- a/codex-cli/scripts/build_npm_package.py +++ b/codex-cli/scripts/build_npm_package.py @@ -326,9 +326,7 @@ def compute_platform_package_version(version: str, platform_tag: str) -> str: def run_command(cmd: list[str], cwd: Path | None = None) -> None: print("+", " ".join(cmd), flush=True) - env = os.environ.copy() - env.setdefault("CI", "true") - subprocess.run(cmd, cwd=cwd, env=env, check=True) + subprocess.run(cmd, cwd=cwd, check=True) def stage_codex_sdk_sources(staging_dir: Path) -> None: diff --git a/codex-cli/scripts/install_native_deps.py b/codex-cli/scripts/install_native_deps.py deleted file mode 100755 index 27ffc5d8408..00000000000 --- a/codex-cli/scripts/install_native_deps.py +++ /dev/null @@ -1,420 +0,0 @@ -#!/usr/bin/env python3 -"""Install Codex package archives and native helper binaries.""" - -import argparse -from contextlib import contextmanager -import os -import shutil -import subprocess -import tarfile -import tempfile -from dataclasses import dataclass -from concurrent.futures import ThreadPoolExecutor, as_completed -from pathlib import Path -from typing import Sequence - -SCRIPT_DIR = Path(__file__).resolve().parent -CODEX_CLI_ROOT = SCRIPT_DIR.parent -DEFAULT_WORKFLOW_URL = "https://github.com/openai/codex/actions/runs/26201494185" # rust-v0.133.0-alpha.4 -GITHUB_REPO = "openai/codex" -VENDOR_DIR_NAME = "vendor" -BINARY_TARGETS = ( - "x86_64-unknown-linux-musl", - "aarch64-unknown-linux-musl", - "x86_64-apple-darwin", - "aarch64-apple-darwin", - "x86_64-pc-windows-msvc", - "aarch64-pc-windows-msvc", -) -CODEX_PACKAGE_COMPONENT = "codex-package" - - -@dataclass(frozen=True) -class BinaryComponent: - artifact_prefix: str # matches the artifact filename prefix (e.g. codex-.zst) - dest_dir: str # directory under vendor// where the binary is installed - binary_basename: str # executable name inside dest_dir (before optional .exe) - - -@dataclass(frozen=True) -class WorkflowArtifact: - name: str - size_in_bytes: int - - -BINARY_COMPONENTS = { - "codex-responses-api-proxy": BinaryComponent( - artifact_prefix="codex-responses-api-proxy", - dest_dir="codex-responses-api-proxy", - binary_basename="codex-responses-api-proxy", - ), -} - - -def _gha_enabled() -> bool: - # GitHub Actions supports "workflow commands" (e.g. ::group:: / ::error::) that make logs - # much easier to scan: groups collapse noisy sections and error annotations surface the - # failure in the UI without changing the actual exception/traceback output. - return os.environ.get("GITHUB_ACTIONS") == "true" - - -def _gha_escape(value: str) -> str: - # Workflow commands require percent/newline escaping. - return value.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") - - -def _gha_error(*, title: str, message: str) -> None: - # Emit a GitHub Actions error annotation. This does not replace stdout/stderr logs; it just - # adds a prominent summary line to the job UI so the root cause is easier to spot. - if not _gha_enabled(): - return - print( - f"::error title={_gha_escape(title)}::{_gha_escape(message)}", - flush=True, - ) - - -@contextmanager -def _gha_group(title: str): - # Wrap a block in a collapsible log group on GitHub Actions. Outside of GHA this is a no-op - # so local output remains unchanged. - if _gha_enabled(): - print(f"::group::{_gha_escape(title)}", flush=True) - try: - yield - finally: - if _gha_enabled(): - print("::endgroup::", flush=True) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Install native Codex binaries.") - parser.add_argument( - "--workflow-url", - help=( - "GitHub Actions workflow URL that produced the artifacts. Defaults to a " - "known good run when omitted." - ), - ) - parser.add_argument( - "--component", - dest="components", - action="append", - choices=tuple([CODEX_PACKAGE_COMPONENT, *BINARY_COMPONENTS]), - help=( - "Limit installation to the specified components." - " May be repeated. Defaults to codex-package and codex-responses-api-proxy." - ), - ) - parser.add_argument( - "--artifacts-dir", - type=Path, - help=( - "Directory used to cache downloaded workflow artifacts. Defaults to a " - "temporary directory." - ), - ) - parser.add_argument( - "root", - nargs="?", - type=Path, - help=( - "Directory containing package.json for the staged package. If omitted, the " - "repository checkout is used." - ), - ) - return parser.parse_args() - - -def main() -> int: - args = parse_args() - - codex_cli_root = (args.root or CODEX_CLI_ROOT).resolve() - vendor_dir = codex_cli_root / VENDOR_DIR_NAME - vendor_dir.mkdir(parents=True, exist_ok=True) - - components = args.components or [CODEX_PACKAGE_COMPONENT, "codex-responses-api-proxy"] - - workflow_override = (args.workflow_url or "").strip() - workflow_url = workflow_override or DEFAULT_WORKFLOW_URL - - workflow_id = workflow_url.rstrip("/").split("/")[-1] - print(f"Downloading native artifacts from workflow {workflow_id}...", flush=True) - - with _gha_group(f"Download native artifacts from workflow {workflow_id}"): - if args.artifacts_dir is not None: - artifacts_dir = args.artifacts_dir.resolve() - artifacts_dir.mkdir(parents=True, exist_ok=True) - install_from_workflow_artifacts(workflow_id, artifacts_dir, components, vendor_dir) - else: - with tempfile.TemporaryDirectory(prefix="codex-native-artifacts-") as artifacts_dir_str: - artifacts_dir = Path(artifacts_dir_str) - install_from_workflow_artifacts( - workflow_id, - artifacts_dir, - components, - vendor_dir, - ) - - print(f"Installed native dependencies into {vendor_dir}", flush=True) - return 0 - - -def install_from_workflow_artifacts( - workflow_id: str, - artifacts_dir: Path, - components: Sequence[str], - vendor_dir: Path, -) -> None: - artifact_names = select_target_artifacts(workflow_id, components) - _download_artifacts(workflow_id, artifacts_dir, artifact_names) - if CODEX_PACKAGE_COMPONENT in components: - install_codex_package_archives(artifacts_dir, vendor_dir, BINARY_TARGETS) - install_binary_components( - artifacts_dir, - vendor_dir, - [BINARY_COMPONENTS[name] for name in components if name in BINARY_COMPONENTS], - ) - - -def select_target_artifacts( - workflow_id: str, - components: Sequence[str], -) -> list[WorkflowArtifact]: - needs_target_artifacts = CODEX_PACKAGE_COMPONENT in components or any( - component in BINARY_COMPONENTS for component in components - ) - if not needs_target_artifacts: - return [] - - artifacts_by_name = { - artifact.name: artifact for artifact in list_workflow_artifacts(workflow_id) - } - selected_artifacts: list[WorkflowArtifact] = [] - for target in BINARY_TARGETS: - for artifact_name in [target, f"{target}-unsigned"]: - artifact = artifacts_by_name.get(artifact_name) - if artifact is not None: - selected_artifacts.append(artifact) - break - else: - raise FileNotFoundError( - f"Expected workflow artifact not found for target {target}" - ) - - return selected_artifacts - - -def list_workflow_artifacts(workflow_id: str) -> list[WorkflowArtifact]: - stdout = subprocess.check_output( - [ - "gh", - "api", - f"repos/{GITHUB_REPO}/actions/runs/{workflow_id}/artifacts", - "--paginate", - "--jq", - ".artifacts[] | [.name, .size_in_bytes] | @tsv", - ], - text=True, - ) - artifacts: list[WorkflowArtifact] = [] - for line in stdout.splitlines(): - name, size_in_bytes = line.split("\t", 1) - artifacts.append(WorkflowArtifact(name=name, size_in_bytes=int(size_in_bytes))) - return artifacts - - -def install_codex_package_archives( - artifacts_dir: Path, - vendor_dir: Path, - targets: Sequence[str], -) -> None: - targets = list(targets) - if not targets: - return - - print( - "Installing Codex package archives for targets: " + ", ".join(targets), - flush=True, - ) - max_workers = min(len(targets), max(1, (os.cpu_count() or 1))) - with ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = { - executor.submit( - _install_single_codex_package_archive, - artifacts_dir, - vendor_dir, - target, - ): target - for target in targets - } - for future in as_completed(futures): - installed_path = future.result() - print(f" installed {installed_path}", flush=True) - - -def _install_single_codex_package_archive( - artifacts_dir: Path, - vendor_dir: Path, - target: str, -) -> Path: - artifact_subdir = artifact_dir_for_target(artifacts_dir, target) - archive_path = artifact_subdir / f"codex-package-{target}.tar.gz" - if not archive_path.exists(): - raise FileNotFoundError(f"Expected package archive not found: {archive_path}") - - dest_dir = vendor_dir / target - if dest_dir.exists(): - shutil.rmtree(dest_dir) - dest_dir.mkdir(parents=True, exist_ok=True) - - with tarfile.open(archive_path, "r:gz") as archive: - archive.extractall(dest_dir, filter="data") - - return dest_dir - - -def _download_artifacts( - workflow_id: str, - dest_dir: Path, - artifacts: Sequence[WorkflowArtifact], -) -> None: - total_bytes = sum(artifact.size_in_bytes for artifact in artifacts) - print( - f"Downloading {len(artifacts)} artifacts ({format_bytes(total_bytes)})", - flush=True, - ) - for artifact in artifacts: - artifact_dir = dest_dir / artifact.name - if artifact_dir.is_dir() and any(artifact_dir.iterdir()): - print( - f" using cached {artifact.name} ({format_bytes(artifact.size_in_bytes)})", - flush=True, - ) - continue - - artifact_dir.mkdir(parents=True, exist_ok=True) - print( - f" downloading {artifact.name} ({format_bytes(artifact.size_in_bytes)})", - flush=True, - ) - cmd = [ - "gh", - "run", - "download", - "--name", - artifact.name, - "--dir", - str(artifact_dir), - "--repo", - GITHUB_REPO, - workflow_id, - ] - subprocess.check_call(cmd) - - -def install_binary_components( - artifacts_dir: Path, - vendor_dir: Path, - selected_components: Sequence[BinaryComponent], -) -> None: - if not selected_components: - return - - for component in selected_components: - component_targets = list(BINARY_TARGETS) - - print( - f"Installing {component.binary_basename} binaries for targets: " - + ", ".join(component_targets), - flush=True, - ) - max_workers = min(len(component_targets), max(1, (os.cpu_count() or 1))) - with ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = { - executor.submit( - _install_single_binary, - artifacts_dir, - vendor_dir, - target, - component, - ): target - for target in component_targets - } - for future in as_completed(futures): - installed_path = future.result() - print(f" installed {installed_path}", flush=True) - - -def format_bytes(size_in_bytes: int) -> str: - value = float(size_in_bytes) - for unit in ["B", "KiB", "MiB"]: - if value < 1024: - return f"{value:.1f} {unit}" - value /= 1024 - return f"{value:.1f} GiB" - - -def _install_single_binary( - artifacts_dir: Path, - vendor_dir: Path, - target: str, - component: BinaryComponent, -) -> Path: - artifact_subdir = artifact_dir_for_target(artifacts_dir, target) - archive_path = binary_archive_path(artifact_subdir, component.artifact_prefix, target) - - dest_dir = vendor_dir / target / component.dest_dir - dest_dir.mkdir(parents=True, exist_ok=True) - - binary_name = ( - f"{component.binary_basename}.exe" if "windows" in target else component.binary_basename - ) - dest = dest_dir / binary_name - dest.unlink(missing_ok=True) - extract_zstd_archive(archive_path, dest) - if "windows" not in target: - dest.chmod(0o755) - return dest - - -def _archive_name_for_target(artifact_prefix: str, target: str) -> str: - if "windows" in target: - return f"{artifact_prefix}-{target}.exe.zst" - return f"{artifact_prefix}-{target}.zst" - - -def binary_archive_path(artifact_dir: Path, artifact_prefix: str, target: str) -> Path: - archive_names = [_archive_name_for_target(artifact_prefix, target)] - if artifact_dir.name == f"{target}-unsigned": - archive_names.append(_archive_name_for_target(artifact_prefix, f"{target}-unsigned")) - - for archive_name in archive_names: - archive_path = artifact_dir / archive_name - if archive_path.exists(): - return archive_path - - raise FileNotFoundError(f"Expected artifact not found: {artifact_dir / archive_names[0]}") - - -def artifact_dir_for_target(artifacts_dir: Path, target: str) -> Path: - for artifact_name in [target, f"{target}-unsigned"]: - artifact_dir = artifacts_dir / artifact_name - if artifact_dir.is_dir(): - return artifact_dir - - return artifacts_dir / target - - -def extract_zstd_archive(archive_path: Path, dest: Path) -> None: - dest.parent.mkdir(parents=True, exist_ok=True) - - output_path = archive_path.parent / dest.name - subprocess.check_call(["zstd", "-f", "-d", str(archive_path), "-o", str(output_path)]) - shutil.move(str(output_path), dest) - - -if __name__ == "__main__": - import sys - - sys.exit(main()) diff --git a/scripts/stage_npm_packages.py b/scripts/stage_npm_packages.py index 3af9cfe6fba..d0cfccf37f2 100755 --- a/scripts/stage_npm_packages.py +++ b/scripts/stage_npm_packages.py @@ -2,20 +2,32 @@ """Stage one or more Codex npm packages for release.""" import argparse +from concurrent.futures import ThreadPoolExecutor, as_completed +from contextlib import contextmanager +from dataclasses import dataclass import importlib.util import json import os import shutil import subprocess +import tarfile import tempfile from pathlib import Path +from typing import Sequence REPO_ROOT = Path(__file__).resolve().parent.parent BUILD_SCRIPT = REPO_ROOT / "codex-cli" / "scripts" / "build_npm_package.py" -INSTALL_NATIVE_DEPS = REPO_ROOT / "codex-cli" / "scripts" / "install_native_deps.py" WORKFLOW_NAME = ".github/workflows/rust-release.yml" GITHUB_REPO = "openai/codex" +BINARY_TARGETS = ( + "x86_64-unknown-linux-musl", + "aarch64-unknown-linux-musl", + "x86_64-apple-darwin", + "aarch64-apple-darwin", + "x86_64-pc-windows-msvc", + "aarch64-pc-windows-msvc", +) _SPEC = importlib.util.spec_from_file_location("codex_build_npm_package", BUILD_SCRIPT) if _SPEC is None or _SPEC.loader is None: @@ -25,6 +37,48 @@ PACKAGE_NATIVE_COMPONENTS = getattr(_BUILD_MODULE, "PACKAGE_NATIVE_COMPONENTS", {}) PACKAGE_EXPANSIONS = getattr(_BUILD_MODULE, "PACKAGE_EXPANSIONS", {}) CODEX_PLATFORM_PACKAGES = getattr(_BUILD_MODULE, "CODEX_PLATFORM_PACKAGES", {}) +CODEX_PACKAGE_COMPONENT = getattr(_BUILD_MODULE, "CODEX_PACKAGE_COMPONENT", "codex-package") + + +@dataclass(frozen=True) +class BinaryComponent: + artifact_prefix: str + dest_dir: str + binary_basename: str + + +@dataclass(frozen=True) +class WorkflowArtifact: + name: str + size_in_bytes: int + + +BINARY_COMPONENTS = { + "codex-responses-api-proxy": BinaryComponent( + artifact_prefix="codex-responses-api-proxy", + dest_dir="codex-responses-api-proxy", + binary_basename="codex-responses-api-proxy", + ), +} + + +def _gha_enabled() -> bool: + return os.environ.get("GITHUB_ACTIONS") == "true" + + +def _gha_escape(value: str) -> str: + return value.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + + +@contextmanager +def _gha_group(title: str): + if _gha_enabled(): + print(f"::group::{_gha_escape(title)}", flush=True) + try: + yield + finally: + if _gha_enabled(): + print("::endgroup::", flush=True) def parse_args() -> argparse.Namespace: @@ -126,17 +180,271 @@ def install_native_components( if not components: return - cmd = [ - str(INSTALL_NATIVE_DEPS), - "--workflow-url", - workflow_url, - "--artifacts-dir", - str(artifacts_dir), - ] - for component in sorted(components): - cmd.extend(["--component", component]) - cmd.append(str(vendor_root)) - run_command(cmd) + vendor_dir = vendor_root / "vendor" + vendor_dir.mkdir(parents=True, exist_ok=True) + + workflow_id = workflow_url.rstrip("/").split("/")[-1] + print(f"Downloading native artifacts from workflow {workflow_id}...", flush=True) + with _gha_group(f"Download native artifacts from workflow {workflow_id}"): + artifacts_dir.mkdir(parents=True, exist_ok=True) + install_from_workflow_artifacts( + workflow_id, + artifacts_dir, + sorted(components), + vendor_dir, + ) + print(f"Installed native dependencies into {vendor_dir}", flush=True) + + +def install_from_workflow_artifacts( + workflow_id: str, + artifacts_dir: Path, + components: Sequence[str], + vendor_dir: Path, +) -> None: + artifacts = select_target_artifacts(workflow_id, components) + download_artifacts(workflow_id, artifacts_dir, artifacts) + if CODEX_PACKAGE_COMPONENT in components: + install_codex_package_archives(artifacts_dir, vendor_dir, BINARY_TARGETS) + install_binary_components( + artifacts_dir, + vendor_dir, + [BINARY_COMPONENTS[name] for name in components if name in BINARY_COMPONENTS], + ) + + +def select_target_artifacts( + workflow_id: str, + components: Sequence[str], +) -> list[WorkflowArtifact]: + needs_target_artifacts = CODEX_PACKAGE_COMPONENT in components or any( + component in BINARY_COMPONENTS for component in components + ) + if not needs_target_artifacts: + return [] + + artifacts_by_name = { + artifact.name: artifact for artifact in list_workflow_artifacts(workflow_id) + } + selected_artifacts: list[WorkflowArtifact] = [] + for target in BINARY_TARGETS: + for artifact_name in [target, f"{target}-unsigned"]: + artifact = artifacts_by_name.get(artifact_name) + if artifact is not None: + selected_artifacts.append(artifact) + break + else: + raise FileNotFoundError( + f"Expected workflow artifact not found for target {target}" + ) + + return selected_artifacts + + +def list_workflow_artifacts(workflow_id: str) -> list[WorkflowArtifact]: + stdout = subprocess.check_output( + [ + "gh", + "api", + f"repos/{GITHUB_REPO}/actions/runs/{workflow_id}/artifacts", + "--paginate", + "--jq", + ".artifacts[] | [.name, .size_in_bytes] | @tsv", + ], + text=True, + ) + artifacts: list[WorkflowArtifact] = [] + for line in stdout.splitlines(): + name, size_in_bytes = line.split("\t", 1) + artifacts.append(WorkflowArtifact(name=name, size_in_bytes=int(size_in_bytes))) + return artifacts + + +def download_artifacts( + workflow_id: str, + dest_dir: Path, + artifacts: Sequence[WorkflowArtifact], +) -> None: + total_bytes = sum(artifact.size_in_bytes for artifact in artifacts) + print( + f"Downloading {len(artifacts)} artifacts ({format_bytes(total_bytes)})", + flush=True, + ) + for artifact in artifacts: + artifact_dir = dest_dir / artifact.name + if artifact_dir.is_dir() and any(artifact_dir.iterdir()): + print( + f" using cached {artifact.name} ({format_bytes(artifact.size_in_bytes)})", + flush=True, + ) + continue + + artifact_dir.mkdir(parents=True, exist_ok=True) + print( + f" downloading {artifact.name} ({format_bytes(artifact.size_in_bytes)})", + flush=True, + ) + subprocess.check_call( + [ + "gh", + "run", + "download", + "--name", + artifact.name, + "--dir", + str(artifact_dir), + "--repo", + GITHUB_REPO, + workflow_id, + ] + ) + + +def install_codex_package_archives( + artifacts_dir: Path, + vendor_dir: Path, + targets: Sequence[str], +) -> None: + if not targets: + return + + print( + "Installing Codex package archives for targets: " + ", ".join(targets), + flush=True, + ) + max_workers = min(len(targets), max(1, (os.cpu_count() or 1))) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = { + executor.submit( + install_single_codex_package_archive, + artifacts_dir, + vendor_dir, + target, + ): target + for target in targets + } + for future in as_completed(futures): + installed_path = future.result() + print(f" installed {installed_path}", flush=True) + + +def install_single_codex_package_archive( + artifacts_dir: Path, + vendor_dir: Path, + target: str, +) -> Path: + artifact_subdir = artifact_dir_for_target(artifacts_dir, target) + archive_path = artifact_subdir / f"codex-package-{target}.tar.gz" + if not archive_path.exists(): + raise FileNotFoundError(f"Expected package archive not found: {archive_path}") + + dest_dir = vendor_dir / target + if dest_dir.exists(): + shutil.rmtree(dest_dir) + dest_dir.mkdir(parents=True, exist_ok=True) + + with tarfile.open(archive_path, "r:gz") as archive: + archive.extractall(dest_dir, filter="data") + + return dest_dir + + +def install_binary_components( + artifacts_dir: Path, + vendor_dir: Path, + selected_components: Sequence[BinaryComponent], +) -> None: + for component in selected_components: + component_targets = list(BINARY_TARGETS) + + print( + f"Installing {component.binary_basename} binaries for targets: " + + ", ".join(component_targets), + flush=True, + ) + max_workers = min(len(component_targets), max(1, (os.cpu_count() or 1))) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = { + executor.submit( + install_single_binary, + artifacts_dir, + vendor_dir, + target, + component, + ): target + for target in component_targets + } + for future in as_completed(futures): + installed_path = future.result() + print(f" installed {installed_path}", flush=True) + + +def install_single_binary( + artifacts_dir: Path, + vendor_dir: Path, + target: str, + component: BinaryComponent, +) -> Path: + artifact_subdir = artifact_dir_for_target(artifacts_dir, target) + archive_path = binary_archive_path(artifact_subdir, component.artifact_prefix, target) + + dest_dir = vendor_dir / target / component.dest_dir + dest_dir.mkdir(parents=True, exist_ok=True) + + binary_name = ( + f"{component.binary_basename}.exe" if "windows" in target else component.binary_basename + ) + dest = dest_dir / binary_name + dest.unlink(missing_ok=True) + extract_zstd_archive(archive_path, dest) + if "windows" not in target: + dest.chmod(0o755) + return dest + + +def binary_archive_path(artifact_dir: Path, artifact_prefix: str, target: str) -> Path: + archive_names = [archive_name_for_target(artifact_prefix, target)] + if artifact_dir.name == f"{target}-unsigned": + archive_names.append(archive_name_for_target(artifact_prefix, f"{target}-unsigned")) + + for archive_name in archive_names: + archive_path = artifact_dir / archive_name + if archive_path.exists(): + return archive_path + + raise FileNotFoundError(f"Expected artifact not found: {artifact_dir / archive_names[0]}") + + +def archive_name_for_target(artifact_prefix: str, target: str) -> str: + if "windows" in target: + return f"{artifact_prefix}-{target}.exe.zst" + return f"{artifact_prefix}-{target}.zst" + + +def artifact_dir_for_target(artifacts_dir: Path, target: str) -> Path: + for artifact_name in [target, f"{target}-unsigned"]: + artifact_dir = artifacts_dir / artifact_name + if artifact_dir.is_dir(): + return artifact_dir + + return artifacts_dir / target + + +def extract_zstd_archive(archive_path: Path, dest: Path) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + + output_path = archive_path.parent / dest.name + subprocess.check_call(["zstd", "-f", "-d", str(archive_path), "-o", str(output_path)]) + shutil.move(str(output_path), dest) + + +def format_bytes(size_in_bytes: int) -> str: + value = float(size_in_bytes) + for unit in ["B", "KiB", "MiB"]: + if value < 1024: + return f"{value:.1f} {unit}" + value /= 1024 + return f"{value:.1f} GiB" def run_command(cmd: list[str]) -> None: