From 73939f36461799cb812569e76d2ccd7507bb4933 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 9 Aug 2026 20:49:19 +0000 Subject: [PATCH] fix(release): complete hosted binary validation FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: Codex:GPT-5 [Codex] --- .github/workflows/release.yml | 11 ++-- docs/USAGE.md | 10 ++-- scripts/build-cpu-release.sh | 12 +++-- scripts/build-linux-accelerator-release.sh | 9 ++++ scripts/build-macos-release.sh | 4 +- scripts/install-intel-sde.sh | 25 +++++++++ scripts/run-cpu-release-gates.py | 53 +++++++++++++------ tests/scripts/test_cpu_release_gates.py | 50 +++++++++++++++-- .../test_release_accelerator_metadata.py | 7 +++ tests/scripts/test_release_macos_metadata.py | 6 +++ tests/scripts/test_release_pipeline.py | 8 ++- 11 files changed, 162 insertions(+), 33 deletions(-) create mode 100755 scripts/install-intel-sde.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8217d2de0..13503d34c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -58,6 +58,7 @@ jobs: run: | sudo apt-get update sudo apt-get install --yes qemu-user + scripts/install-intel-sde.sh "$RUNNER_TEMP/intel-sde" - name: Build, execute, package, and validate adaptive x86 CPU bundle env: EVIDENCE_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} @@ -72,7 +73,8 @@ jobs: scripts/build-cpu-release.sh \ linux-x86_64-glibc-cpu x86_64 stable build-release-cpu-x86 \ "$(getconf GNU_LIBC_VERSION | awk '{print $2}')" \ - "$(command -v qemu-x86_64)" + "$(command -v qemu-x86_64)" \ + "$RUNNER_TEMP/intel-sde/sde64" - name: Upload exact validated x86 CPU asset triplet id: upload uses: actions/upload-artifact@v4 @@ -111,6 +113,7 @@ jobs: scripts/build-cpu-release.sh \ linux-aarch64-glibc-cpu aarch64 stable build-release-cpu-arm64 \ "$(getconf GNU_LIBC_VERSION | awk '{print $2}')" \ + "$(command -v qemu-aarch64)" \ "$(command -v qemu-aarch64)" - name: Upload exact validated Arm CPU asset triplet id: upload @@ -143,13 +146,15 @@ jobs: export SOURCE_DATE_EPOCH version=$(sed -n 's/^project(vllm_cpp VERSION \([^ ]*\).*/\1/p' CMakeLists.txt) test "$version" = "$VERSION" + scripts/install-intel-sde.sh "$RUNNER_TEMP/intel-sde" docker run --rm \ -e EVIDENCE_URL -e SOURCE_DATE_EPOCH -e SOURCE_SHA -e VERSION \ + -v "$RUNNER_TEMP/intel-sde:/intel-sde:ro" \ -v "$GITHUB_WORKSPACE:/src" -w /src alpine:3.22 sh -euxc ' - apk add --no-cache bash binutils build-base cmake file ninja python3 qemu-x86_64 + apk add --no-cache bash binutils build-base cmake file gcompat ninja python3 qemu-x86_64 scripts/build-cpu-release.sh \ linux-x86_64-musl-cpu-static x86_64 experimental-preview \ - build-release-cpu-musl 1.2.5 /usr/bin/qemu-x86_64 + build-release-cpu-musl 1.2.5 /usr/bin/qemu-x86_64 /intel-sde/sde64 ' sudo chown -R "$(id -u):$(id -g)" build-release-cpu-musl - name: Upload exact validated musl CPU asset triplet diff --git a/docs/USAGE.md b/docs/USAGE.md index a5ec7007f..5c00009a0 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -192,9 +192,11 @@ bytes; placing either inside those bytes would create a self-reference. The CPU release helper is the reproducible entry point used by CI. It requires an explicit artifact tuple, architecture, channel, build directory, libc ABI, -and a QEMU userspace emulator. The gate executes every compiled tier under a -feature-rich CPU model, then executes the baseline and proves rich-tier refusal -under a feature-poor model before metadata can be generated: +a feature-poor QEMU userspace emulator, and a feature-rich runner. x86_64 uses +the SHA256-pinned Intel SDE installed by `scripts/install-intel-sde.sh` so the +AVX-512 tier is really executed even when the host lacks AVX-512. The gate then +executes the baseline and proves rich-tier refusal under the feature-poor QEMU +model before metadata can be generated: ```sh SOURCE_SHA=$(git rev-parse HEAD) \ @@ -203,7 +205,7 @@ SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD) \ EVIDENCE_URL=https://github.com/mudler/vllm.cpp/actions/runs/EXAMPLE \ scripts/build-cpu-release.sh \ linux-x86_64-glibc-cpu x86_64 stable build-release-cpu-x86 \ - 2.39 /usr/bin/qemu-x86_64 + 2.39 /usr/bin/qemu-x86_64 /tmp/intel-sde/sde64 ``` The corresponding arm64 tuple is `linux-aarch64-glibc-cpu`. The only literal diff --git a/scripts/build-cpu-release.sh b/scripts/build-cpu-release.sh index c87584727..5d1512d26 100755 --- a/scripts/build-cpu-release.sh +++ b/scripts/build-cpu-release.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash set -euo pipefail -if [[ $# -ne 6 ]]; then - echo "usage: $0 ARTIFACT_ID ARCH CHANNEL BUILD_DIR ABI_VERSION POOR_EMULATOR" >&2 +if [[ $# -ne 7 ]]; then + echo "usage: $0 ARTIFACT_ID ARCH CHANNEL BUILD_DIR ABI_VERSION POOR_EMULATOR RICH_RUNNER" >&2 exit 2 fi @@ -12,6 +12,7 @@ channel=$3 build_dir=$4 abi_version=$5 poor_emulator=$6 +rich_runner=$7 : "${SOURCE_SHA:?SOURCE_SHA is required}" : "${VERSION:?VERSION is required}" : "${EVIDENCE_URL:?EVIDENCE_URL is required}" @@ -51,11 +52,16 @@ metadata_dir="$release_dir/metadata" archive="$release_dir/$artifact_id.tar.gz" mkdir -p "$release_dir" +rich_runner_kind=qemu +if [[ "$arch" == x86_64 ]]; then + rich_runner_kind=intel-sde +fi python3 scripts/run-cpu-release-gates.py \ --arch "$arch" \ --tests-dir "$build_dir/tests" \ --poor-emulator "$poor_emulator" \ - --rich-emulator "$poor_emulator" \ + --rich-runner "$rich_runner" \ + --rich-runner-kind "$rich_runner_kind" \ --rich-cpu max \ --output "$tier_report" \ --evidence-url "$EVIDENCE_URL" diff --git a/scripts/build-linux-accelerator-release.sh b/scripts/build-linux-accelerator-release.sh index 5052e1aec..4a15ae7bf 100755 --- a/scripts/build-linux-accelerator-release.sh +++ b/scripts/build-linux-accelerator-release.sh @@ -95,6 +95,15 @@ python3 scripts/package-server.py \ --stage-dir "$stage_dir" \ --metadata-dir "$metadata_dir" \ --archive "$archive" +if [[ "$backend" == cuda ]]; then + cuda_stub=$(find /usr/local/cuda -type f -path '*/stubs/libcuda.so' -print -quit) + if [[ -z "$cuda_stub" ]]; then + echo "CUDA driver stub is required for archive smoke validation" >&2 + exit 1 + fi + cuda_stub_dir=${cuda_stub%/*} + export LD_LIBRARY_PATH="$cuda_stub_dir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +fi python3 scripts/validate-release-archive.py \ --archive "$archive" \ --checksum "$archive.sha256" \ diff --git a/scripts/build-macos-release.sh b/scripts/build-macos-release.sh index 34759f6e6..ee099690f 100755 --- a/scripts/build-macos-release.sh +++ b/scripts/build-macos-release.sh @@ -18,14 +18,12 @@ mlx_license=${6:-} : "${SOURCE_DATE_EPOCH:?SOURCE_DATE_EPOCH is required}" mlx=OFF -mlx_metadata_args=() if [[ "$artifact_id" == macos-arm64-metal-mlx ]]; then mlx=ON if [[ -z "$mlx_root" || -z "$mlx_version" || -z "$mlx_license" ]]; then echo "MLX artifact requires an exact root, version, and license" >&2 exit 2 fi - mlx_metadata_args=(--mlx-license "$mlx_license") fi cmake -S . -B "$build_dir" -G Ninja \ @@ -68,7 +66,7 @@ python3 scripts/release_macos_metadata.py \ --source-clean \ --abi-version "$(sw_vers -productVersion)" \ --mlx-version "$mlx_version" \ - "${mlx_metadata_args[@]}" \ + --mlx-license "$mlx_license" \ --compiler "$compiler" \ --toolchain "$toolchain" \ --evidence-url "$EVIDENCE_URL" diff --git a/scripts/install-intel-sde.sh b/scripts/install-intel-sde.sh new file mode 100755 index 000000000..ff06b5fa3 --- /dev/null +++ b/scripts/install-intel-sde.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "usage: $0 DESTINATION" >&2 + exit 2 +fi + +destination=$1 +archive_name=sde-external-10.13.1-2026-07-28-lin.tar.xz +archive_sha256=94e97d623fec54385686e1e7ba65ebc9941748c05ee451423948334892bf2b50 +archive_url=https://downloadmirror.intel.com/924984/$archive_name + +if [[ -e "$destination" ]]; then + echo "Intel SDE destination already exists: $destination" >&2 + exit 1 +fi + +mkdir -p "$destination" +archive="$destination/$archive_name" +curl --fail --location --retry 3 --output "$archive" "$archive_url" +printf '%s %s\n' "$archive_sha256" "$archive" | sha256sum --check +tar --extract --xz --file "$archive" --directory "$destination" --strip-components=1 +rm -f -- "$archive" +test -x "$destination/sde64" diff --git a/scripts/run-cpu-release-gates.py b/scripts/run-cpu-release-gates.py index 64504e47d..38d758824 100755 --- a/scripts/run-cpu-release-gates.py +++ b/scripts/run-cpu-release-gates.py @@ -18,6 +18,9 @@ "VT_CPU_QUANT_MMLA", "VT_CPU_QUANT_REPACK", ) +MATMUL_GATE_ARG = ( + "--test-case=elementwise CPU GEMM: row-strided activation stays bit-exact" +) class GateError(RuntimeError): @@ -29,7 +32,8 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--arch", choices=("x86_64", "aarch64"), required=True) parser.add_argument("--tests-dir", type=Path, required=True) parser.add_argument("--poor-emulator", type=Path, required=True) - parser.add_argument("--rich-emulator", type=Path) + parser.add_argument("--rich-runner", type=Path) + parser.add_argument("--rich-runner-kind", choices=("qemu", "intel-sde")) parser.add_argument("--rich-cpu", default="max") parser.add_argument("--output", type=Path, required=True) parser.add_argument("--evidence-url", required=True) @@ -43,22 +47,32 @@ def executable(tests_dir: Path, name: str) -> Path: return path -def display_command(prefix: list[str], env_values: dict[str, str], binary: Path) -> str: +def test_args(binary: Path, targeted: bool) -> list[str]: + return [MATMUL_GATE_ARG] if targeted and binary.name == "test_ops_matmul_elem" else [] + + +def display_command( + prefix: list[str], env_values: dict[str, str], binary: Path, targeted: bool +) -> str: assignments = " ".join(f"{name}={shlex.quote(value)}" for name, value in env_values.items()) - command = shlex.join([*prefix, str(binary)]) + command = shlex.join([*prefix, str(binary), *test_args(binary, targeted)]) return f"{assignments} {command}".strip() def run_command( - prefix: list[str], env_values: dict[str, str], binary: Path, expect_failure: bool = False + prefix: list[str], + env_values: dict[str, str], + binary: Path, + expect_failure: bool = False, + targeted: bool = False, ) -> tuple[str, str]: - command = display_command(prefix, env_values, binary) + command = display_command(prefix, env_values, binary, targeted) environment = os.environ.copy() for name in TIER_ENV: environment.pop(name, None) environment.update(env_values) result = subprocess.run( - [*prefix, str(binary)], + [*prefix, str(binary), *test_args(binary, targeted)], env=environment, text=True, stdout=subprocess.PIPE, @@ -100,11 +114,16 @@ def run_group( def gate(args: argparse.Namespace) -> dict[str, object]: tests_dir = args.tests_dir.resolve() matmul = executable(tests_dir, "test_ops_matmul_elem") - rich_prefix = ( - [str(args.rich_emulator.resolve()), "-cpu", args.rich_cpu] - if args.rich_emulator is not None - else [] - ) + if (args.rich_runner is None) != (args.rich_runner_kind is None): + raise GateError("rich runner and kind must be provided together") + rich_prefix: list[str] = [] + if args.rich_runner is not None: + rich_runner = str(args.rich_runner.resolve()) + rich_prefix = ( + [rich_runner, "-cpu", args.rich_cpu] + if args.rich_runner_kind == "qemu" + else [rich_runner, "-skx", "--"] + ) poor_model = "Nehalem" if args.arch == "x86_64" else "cortex-a53" poor_prefix = [str(args.poor_emulator.resolve()), "-cpu", poor_model] tiers: dict[str, dict[str, str]] = {} @@ -124,13 +143,15 @@ def gate(args: argparse.Namespace) -> dict[str, object]: {"VT_CPU_MATMUL_TIER": "portable"}, {"VT_CPU_MATMUL_TIER": "sse2"}, ): - command, _ = run_command(poor_prefix, env_values, matmul) + command, _ = run_command(poor_prefix, env_values, matmul, targeted=True) commands.append(command) for env_values in ( {"VT_CPU_MATMUL_TIER": "sse2+f16c"}, {"VT_CPU_MATMUL_TIER": "avx2"}, ): - command, _ = run_command(poor_prefix, env_values, matmul, expect_failure=True) + command, _ = run_command( + poor_prefix, env_values, matmul, expect_failure=True, targeted=True + ) commands.append(command) selected_tier = "avx512f" else: @@ -168,7 +189,7 @@ def gate(args: argparse.Namespace) -> dict[str, object]: ({"VT_CPU_QUANT_REPACK": "portable"}, quant_repack), ) for env_values, binary in poor_pass: - command, _ = run_command(poor_prefix, env_values, binary) + command, _ = run_command(poor_prefix, env_values, binary, targeted=True) commands.append(command) poor_refuse = ( ({"VT_CPU_Q8_DOT": "sdot"}, quant_dot), @@ -176,7 +197,9 @@ def gate(args: argparse.Namespace) -> dict[str, object]: ({"VT_CPU_QUANT_REPACK": "i8mm"}, quant_repack), ) for env_values, binary in poor_refuse: - command, _ = run_command(poor_prefix, env_values, binary, expect_failure=True) + command, _ = run_command( + poor_prefix, env_values, binary, expect_failure=True, targeted=True + ) commands.append(command) selected_tier = "i8mm" diff --git a/tests/scripts/test_cpu_release_gates.py b/tests/scripts/test_cpu_release_gates.py index d5a28760d..4bcaf8981 100755 --- a/tests/scripts/test_cpu_release_gates.py +++ b/tests/scripts/test_cpu_release_gates.py @@ -34,6 +34,14 @@ """ +FAKE_SDE = """#!/bin/sh +test "$1" = -skx +test "$2" = -- +shift 2 +exec "$@" +""" + + class CpuReleaseGatesContract(unittest.TestCase): def fixture(self, scratch: Path) -> tuple[Path, Path]: tests = scratch / "tests" @@ -52,9 +60,15 @@ def fixture(self, scratch: Path) -> tuple[Path, Path]: emulator.chmod(0o755) return tests, emulator - def invoke(self, tests: Path, emulator: Path, arch: str, output: Path): - return subprocess.run( - [ + def invoke( + self, + tests: Path, + emulator: Path, + arch: str, + output: Path, + rich_runner: Path | None = None, + ): + argv = [ sys.executable, str(TOOL), "--arch", @@ -67,13 +81,31 @@ def invoke(self, tests: Path, emulator: Path, arch: str, output: Path): str(output), "--evidence-url", "https://github.com/mudler/vllm.cpp/actions/runs/1", - ], + ] + if rich_runner is not None: + argv.extend(["--rich-runner", str(rich_runner), "--rich-runner-kind", "intel-sde"]) + return subprocess.run( + argv, text=True, capture_output=True, env={**os.environ, "VLLM_CPP_RELEASE_GATE_TEST": "1"}, check=False, ) + def test_x86_rich_tiers_use_the_explicit_intel_sde_prefix(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + tests, emulator = self.fixture(Path(temporary)) + rich_runner = Path(temporary) / "sde64" + rich_runner.write_text(FAKE_SDE, encoding="utf-8") + rich_runner.chmod(0o755) + output = Path(temporary) / "report.json" + result = self.invoke(tests, emulator, "x86_64", output, rich_runner) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + report = json.loads(output.read_text()) + self.assertTrue( + all("sde64 -skx --" in tier["command"] for tier in report["tiers"].values()) + ) + def test_x86_executes_all_tiers_and_poor_host_refusal(self) -> None: with tempfile.TemporaryDirectory() as temporary: tests, emulator = self.fixture(Path(temporary)) @@ -88,6 +120,16 @@ def test_x86_executes_all_tiers_and_poor_host_refusal(self) -> None: self.assertEqual({row["state"] for row in report["tiers"].values()}, {"passed"}) self.assertTrue(any("Nehalem" in command for command in report["commands"])) self.assertTrue(any("expect-refusal" in command for command in report["commands"])) + poor_commands = [command for command in report["commands"] if "Nehalem" in command] + self.assertTrue( + all("--test-case=elementwise CPU GEMM:" in command for command in poor_commands) + ) + self.assertTrue( + all( + "--test-case=elementwise CPU GEMM:" not in tier["command"] + for tier in report["tiers"].values() + ) + ) def test_arm_executes_dotprod_i8mm_and_poor_host_refusal(self) -> None: with tempfile.TemporaryDirectory() as temporary: diff --git a/tests/scripts/test_release_accelerator_metadata.py b/tests/scripts/test_release_accelerator_metadata.py index c8819478e..6e6a1346c 100755 --- a/tests/scripts/test_release_accelerator_metadata.py +++ b/tests/scripts/test_release_accelerator_metadata.py @@ -14,6 +14,7 @@ ROOT = Path(__file__).resolve().parents[2] TOOL = ROOT / "scripts/release_accelerator_metadata.py" +BUILD_SCRIPT = ROOT / "scripts/build-linux-accelerator-release.sh" SHA = "0123456789abcdef0123456789abcdef01234567" SMS = ["80", "86", "87", "89", "90a", "100a", "103a", "110", "120a", "121a"] @@ -110,6 +111,12 @@ def test_cuda_refuses_partial_sm_or_disabled_triton_cache(self) -> None: self.tool.prepare_accelerator_metadata(args) (args.build_dir / "CMakeCache.txt").write_text(cache, encoding="utf-8") + def test_cuda_archive_smoke_resolves_only_the_external_driver_stub(self) -> None: + script = BUILD_SCRIPT.read_text(encoding="utf-8") + self.assertIn("*/stubs/libcuda.so", script) + self.assertIn("CUDA driver stub is required for archive smoke validation", script) + self.assertIn('export LD_LIBRARY_PATH="$cuda_stub_dir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"', script) + if __name__ == "__main__": unittest.main() diff --git a/tests/scripts/test_release_macos_metadata.py b/tests/scripts/test_release_macos_metadata.py index 8225f9ffa..9213c8995 100755 --- a/tests/scripts/test_release_macos_metadata.py +++ b/tests/scripts/test_release_macos_metadata.py @@ -15,6 +15,7 @@ ROOT = Path(__file__).resolve().parents[2] TOOL = ROOT / "scripts/release_macos_metadata.py" +BUILD_SCRIPT = ROOT / "scripts/build-macos-release.sh" SHA = "0123456789abcdef0123456789abcdef01234567" @@ -121,6 +122,11 @@ def test_mlx_preview_requires_bundled_versioned_runtime_and_license(self) -> Non with self.assertRaises(ValueError): self.tool.prepare_macos_metadata(args) + def test_native_metal_packaging_is_compatible_with_system_bash(self) -> None: + script = BUILD_SCRIPT.read_text(encoding="utf-8") + self.assertNotIn("mlx_metadata_args", script) + self.assertIn('--mlx-license "$mlx_license"', script) + if __name__ == "__main__": unittest.main() diff --git a/tests/scripts/test_release_pipeline.py b/tests/scripts/test_release_pipeline.py index 19a660534..28bd741dd 100755 --- a/tests/scripts/test_release_pipeline.py +++ b/tests/scripts/test_release_pipeline.py @@ -264,12 +264,18 @@ def test_workflow_has_exact_least_privilege_stage_boundaries(self) -> None: def test_hosted_packagers_resolve_their_runtime_dependencies(self) -> None: workflow = WORKFLOW.read_text(encoding="utf-8") self.assertIn( - "apk add --no-cache bash binutils build-base cmake file ninja python3 qemu-x86_64", + "apk add --no-cache bash binutils build-base cmake file gcompat ninja python3 qemu-x86_64", workflow, ) self.assertNotIn("mlx.__file__", workflow) self.assertIn('d.locate_file("mlx")', workflow) musl = self.checker.job_block(workflow, "cpu_musl") + x86 = self.checker.job_block(workflow, "cpu_x86") + self.assertIn('scripts/install-intel-sde.sh "$RUNNER_TEMP/intel-sde"', x86) + self.assertIn('"$RUNNER_TEMP/intel-sde/sde64"', x86) + self.assertIn('scripts/install-intel-sde.sh "$RUNNER_TEMP/intel-sde"', musl) + self.assertIn('-v "$RUNNER_TEMP/intel-sde:/intel-sde:ro"', musl) + self.assertIn("/intel-sde/sde64", musl) self.assertIn( "SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD)\n" " export SOURCE_DATE_EPOCH",