refactor: updates to the py_extension rule for building C extensions

.agents/skills/merge-pr/SKILL.md

@@ -0,0 +1,14 @@
+---
+name: merge-pr
+description: Merge a pull request into main, monitoring the merge queue, retrying CI flakes, and re-enqueuing if necessary
+---
+
+When the user asks to merge a pull request (e.g., "merge PR <number>", "merge this PR", or monitor its merge):
+
+1. **Enqueue for Merge**: Run `gh pr merge <pr_number> --auto --squash` to enable auto-merge or add the pull request to the merge queue.
+2. **Invoke a Background Shepherd**: Launch a background subagent with the role `Merge PR Shepherd` to continuously watch the PR until it merges.
+3. **Leverage Existing CI Skills**: 
+   - Have the subagent use the **`monitor-ci-results`** skill to watch for CI check failures and generate analysis reports.
+   - Have the subagent use the **`buildkite-retry-job`** skill (`retry_buildkite_jobs.py <pr_number>`) to automatically retry any transient network flakes (e.g., HTTP 504 gateway timeouts, downloader errors).
+4. **Queue Shepherding**: Periodically check `gh pr view <pr_number> --json state,autoMergeRequest`. While `state` is `"OPEN"`, ensure auto-merge is enabled / queued by running `gh pr merge <pr_number> --auto --squash`. If `autoMergeRequest` is null (e.g., ejected from the merge queue due to a CI flake in the temporary queue branch), re-enqueue it for merge by running `gh pr merge <pr_number> --auto --squash` once checks are retried or green.
+5. **Completion Notification**: Once `state` becomes `"MERGED"`, send a high-priority message back to the parent conversation.

.github/workflows/automated_pr_review.yaml

@@ -62,7 +62,7 @@ jobs:
 
       # Upload extracted diff as artifact to pass to Job 2 safely as text data.
       - name: Upload Diff Artifact
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: pr_diff
           path: pr_diff.txt

python/cc/py_extension.bzl

@@ -1,4 +1,8 @@
-"""Public API for py_extension."""
+"""Public API for py_extension.
+
+:::{include} /_includes/experimental_api.md
+:::
+"""
 
 load(
     "//python/private/cc:py_extension_macro.bzl",

python/private/cc/py_extension_macro.bzl

@@ -1,4 +1,8 @@
-"""Macro for creating Python extensions."""
+"""Macro for creating Python extensions.
+
+:::{include} /_includes/experimental_api.md
+:::
+"""
 
 load("@rules_cc//cc:cc_library.bzl", "cc_library")
 load("@rules_cc//cc:cc_shared_library.bzl", "cc_shared_library")
@@ -24,7 +28,7 @@ def py_extension(
         **kwargs):
     """Creates a Python extension module.
 
-    :::{include} /_includes/volatile_api.md
+    :::{include} /_includes/experimental_api.md
     :::
 
     By default, extensions are created within their workspace package directory

python/private/cc/py_extension_rule.bzl

@@ -1,4 +1,8 @@
-"""Implementation of the _py_extension_wrapper rule."""
+"""Implementation of the _py_extension_wrapper rule.
+
+:::{include} /_includes/experimental_api.md
+:::
+"""
 
 load("@bazel_skylib//lib:dicts.bzl", "dicts")
 load("@rules_cc//cc/common:cc_shared_library_info.bzl", "CcSharedLibraryInfo")
@@ -95,8 +99,9 @@ PY_EXTENSION_WRAPPER_ATTRS = dicts.add(
 )
 
 def create_py_extension_wrapper_rule_builder(**kwargs):
-    """Create a rule builder for the wrapper."""
+    """Create a rule builder for the private internal wrapper rule."""
     builder = ruleb.Rule(
+        doc = "Private internal helper rule for py_extension targets.",
         implementation = _py_extension_wrapper_impl,
         attrs = PY_EXTENSION_WRAPPER_ATTRS,
         provides = [PyInfo],
@@ -140,10 +145,9 @@ def _get_platform(ctx):
     """
     py_toolchain = ctx.toolchains[PY_CC_TOOLCHAIN_TYPE]
     py_cc_toolchain = py_toolchain.py_cc_toolchain
-    if hasattr(py_cc_toolchain, "platform_tag") and py_cc_toolchain.platform_tag:
-        return py_cc_toolchain.platform_tag
-
-    fail(
-        "ERROR: Unable to resolve platform_tag from Python C++ toolchain for {self}. " +
-        "Please ensure the active py_cc_toolchain provides a non-empty platform_tag.",
-    )
+    if not py_cc_toolchain.platform_tag:
+        fail(
+            "ERROR: Unable to resolve platform_tag from Python C++ toolchain for {self}. " +
+            "Please ensure the active py_cc_toolchain provides a non-empty platform_tag.",
+        )
+    return py_cc_toolchain.platform_tag

python/private/py_cc_toolchain_info.bzl

@@ -96,16 +96,29 @@ If available, information about C libraries, struct with fields:
     considered private and should be forward along as-is (this better allows
     e.g. `:current_py_cc_headers` to act as the underlying headers target it
     represents).
+""",
+        "platform_machine": """
+:type: str
+
+The [PEP 508](https://peps.python.org/pep-0508/) `platform_machine` marker
+value for the target architecture, e.g. 'x86_64', 'aarch64'.
 """,
         "platform_tag": """\
 :type: str | None
 
-The PEP 3149 / PEP 425 platform tag for extension modules, e.g. 'x86_64-linux-gnu', 'darwin', or 'win_amd64'.
+The PEP 3149 / PEP 425 platform tag for extension modules, e.g.
+'x86_64-linux-gnu', 'darwin', or 'win_amd64'.
 """,
         "python_version": """
 :type: str
 
 The Python Major.Minor version.
+""",
+        "sys_platform": """
+:type: str
+
+The [PEP 508](https://peps.python.org/pep-0508/) `sys_platform` marker value
+for the target OS, e.g. 'linux', 'darwin', 'win32'.
 """,
     },
 )

python/private/py_cc_toolchain_macro.bzl

@@ -14,6 +14,7 @@
 
 """Fronting macro for the py_cc_toolchain rule."""
 
+load("//python/private/pypi:pep508_env.bzl", "platform_machine_select_map", "sys_platform_select_map")
 load(":py_cc_toolchain_rule.bzl", _py_cc_toolchain = "py_cc_toolchain")
 load(":util.bzl", "add_tag")
 
@@ -31,19 +32,11 @@ def py_cc_toolchain(**kwargs):
     #  This tag is added to easily identify usages through other macros.
     add_tag(kwargs, "@rules_python//python:py_cc_toolchain")
 
-    if "os" not in kwargs:
-        kwargs["os"] = select({
-            "@platforms//os:macos": "macos",
-            "@platforms//os:windows": "windows",
-            "//conditions:default": "linux",
-        })
-    if "cpu" not in kwargs:
-        kwargs["cpu"] = select({
-            "@platforms//cpu:aarch64": "aarch64",
-            "@platforms//cpu:x86_32": "x86_32",
-            "//conditions:default": "x86_64",
-        })
-    if "libc" not in kwargs:
+    if kwargs.get("sys_platform") == None:
+        kwargs["sys_platform"] = select(sys_platform_select_map)
+    if kwargs.get("platform_machine") == None:
+        kwargs["platform_machine"] = select(platform_machine_select_map)
+    if kwargs.get("libc") == None:
         kwargs["libc"] = select({
             Label("//python/config_settings:_is_py_linux_libc_musl"): "musl",
             "//conditions:default": "gnu",

python/private/py_cc_toolchain_rule.bzl

@@ -25,8 +25,8 @@ load(":flags.bzl", "FreeThreadedFlag")
 load(":py_cc_toolchain_info.bzl", "PyCcToolchainInfo")
 load(":sentinel_impl.bzl", "SentinelInfo")
 
-def _get_platform_tag(os, cpu, libc):
-    """Derives the PEP 3149 platform tag string based on target OS, CPU, and libc.
+def _get_platform_tag(sys_platform, platform_machine, libc):
+    """Derives the PEP 3149 platform tag string.
 
     Note that these are platform tags for C extension filenames, not
     PEP 425 tags for wheels.
@@ -41,25 +41,25 @@ def _get_platform_tag(os, cpu, libc):
       - https://github.com/python/cpython/commit/3b8124884c3655b4cf2629d741b18c1a38181805
 
     Args:
-        os: Target OS, e.g. "windows", "macos", "linux"
-        cpu: Target CPU architecture, e.g. "x86_64", "aarch64", "x86_32"
+        sys_platform: Target PEP 508 OS marker, e.g. "win32", "darwin", "linux"
+        platform_machine: Target PEP 508 CPU marker, e.g. "x86_64", "aarch64", "x86_32"
         libc: Target C library variant, e.g. "gnu", "musl"
 
     Returns:
         The platform tag, e.g. "x86_64-linux-gnu", "darwin", or "win_amd64"
     """
-    if os == "windows":
-        if cpu == "x86_64":
+    if sys_platform == "win32":
+        if platform_machine in ("x86_64", "amd64"):
             return "win_amd64"
-        if cpu == "aarch64":
+        if platform_machine in ("aarch64", "arm64"):
             return "win_arm64"
         return "win32"
-    if os == "macos":
+    if sys_platform == "darwin":
         return "darwin"
 
-    cpu_val = cpu if cpu else "x86_64"
+    machine_val = platform_machine if platform_machine else "x86_64"
     libc_val = libc if libc else "gnu"
-    return "{}-linux-{}".format(cpu_val, libc_val)
+    return "{}-linux-{}".format(machine_val, libc_val)
 
 def _py_cc_toolchain_impl(ctx):
     if ctx.attr.libs:
@@ -92,14 +92,13 @@ def _py_cc_toolchain_impl(ctx):
         abi_tag = "cpython-{}{}{}".format(version_parts[0], version_parts[1], abi_flags)
 
     platform_tag = _get_platform_tag(
-        os = ctx.attr.os,
-        cpu = ctx.attr.cpu,
+        sys_platform = ctx.attr.sys_platform,
+        platform_machine = ctx.attr.platform_machine,
         libc = ctx.attr.libc,
     )
 
     py_cc_toolchain = PyCcToolchainInfo(
         abi_tag = abi_tag,
-        platform_tag = platform_tag,
         headers = struct(
             providers_map = {
                 "CcInfo": ctx.attr.headers[CcInfo],
@@ -108,7 +107,10 @@ def _py_cc_toolchain_impl(ctx):
         ),
         headers_abi3 = headers_abi3,
         libs = libs,
+        platform_machine = ctx.attr.platform_machine,
+        platform_tag = platform_tag,
         python_version = ctx.attr.python_version,
+        sys_platform = ctx.attr.sys_platform,
     )
     extra_kwargs = {}
     if ctx.attr._visible_for_testing[BuildSettingInfo].value:
@@ -125,10 +127,6 @@ py_cc_toolchain = rule(
             doc = "The ABI tag for extension modules, e.g. 'cpython-311'",
             default = "",
         ),
-        "cpu": attr.string(
-            doc = "Target CPU architecture, e.g. 'x86_64', 'aarch64', 'x86_32'",
-            default = "",
-        ),
         "headers": attr.label(
             doc = ("Target that provides the Python headers. Typically this " +
                    "is a cc_library target."),
@@ -158,14 +156,22 @@ attribute is available or not.
                    "Typically this is a cc_library target of `.so` files."),
             providers = [CcInfo],
         ),
-        "os": attr.string(
-            doc = "Target OS, e.g. 'linux', 'macos', 'windows'",
+        "platform_machine": attr.string(
+            doc = """
+Target architecture as a PEP 508 `platform_machine` marker, e.g. 'x86_64', 'aarch64', 'x86_32'.
+""",
             default = "",
         ),
         "python_version": attr.string(
             doc = "The Major.minor Python version, e.g. 3.11",
             mandatory = True,
         ),
+        "sys_platform": attr.string(
+            doc = """
+Target OS as a PEP 508 `sys_platform` marker, e.g. 'linux', 'darwin', 'win32'.
+""",
+            default = "",
+        ),
         "_py_freethreaded_flag": attr.label(
             default = labels.PY_FREETHREADED,
         ),

tests/cc/py_cc_toolchain/py_cc_toolchain_tests.bzl

@@ -118,8 +118,38 @@ def _test_py_cc_toolchain_impl(env, target):
         matching.str_matches("/libpython3."),
     )
 
+    # ===== Verify PEP 508 platform markers =====
+    toolchain.sys_platform().equals("linux")
+    toolchain.platform_machine().equals("x86_64")
+    toolchain.platform_tag().equals("x86_64-linux-gnu")
+
 _tests.append(_test_py_cc_toolchain)
 
+def _test_custom_pep508_markers(name):
+    py_cc_toolchain(
+        name = name + "_subject",
+        headers = "//tests/support/cc_toolchains:py_headers",
+        platform_machine = "arm64",
+        python_version = "3.11",
+        sys_platform = "darwin",
+    )
+    analysis_test(
+        name = name,
+        target = name + "_subject",
+        impl = _test_custom_pep508_markers_impl,
+    )
+
+def _test_custom_pep508_markers_impl(env, target):
+    toolchain = PyCcToolchainInfoSubject.new(
+        target[platform_common.ToolchainInfo].py_cc_toolchain,
+        meta = env.expect.meta.derive(expr = "py_cc_toolchain_info"),
+    )
+    toolchain.sys_platform().equals("darwin")
+    toolchain.platform_machine().equals("arm64")
+    toolchain.platform_tag().equals("darwin")
+
+_tests.append(_test_custom_pep508_markers)
+
 def _test_libs_optional(name):
     py_cc_toolchain(
         name = name + "_subject",

tests/support/py_cc_toolchain_info_subject.bzl

@@ -21,7 +21,10 @@ def _py_cc_toolchain_info_subject_new(info, *, meta):
         headers = lambda *a, **k: _py_cc_toolchain_info_subject_headers(self, *a, **k),
         headers_abi3 = lambda *a, **k: _py_cc_toolchain_info_subject_headers_abi3(self, *a, **k),
         libs = lambda *a, **k: _py_cc_toolchain_info_subject_libs(self, *a, **k),
+        platform_machine = lambda *a, **k: _py_cc_toolchain_info_subject_platform_machine(self, *a, **k),
+        platform_tag = lambda *a, **k: _py_cc_toolchain_info_subject_platform_tag(self, *a, **k),
         python_version = lambda *a, **k: _py_cc_toolchain_info_subject_python_version(self, *a, **k),
+        sys_platform = lambda *a, **k: _py_cc_toolchain_info_subject_sys_platform(self, *a, **k),
         actual = info,
     )
     self = struct(actual = info, meta = meta)
@@ -54,12 +57,30 @@ def _py_cc_toolchain_info_subject_libs(self):
         ),
     )
 
+def _py_cc_toolchain_info_subject_platform_machine(self):
+    return subjects.str(
+        self.actual.platform_machine,
+        meta = self.meta.derive("platform_machine()"),
+    )
+
+def _py_cc_toolchain_info_subject_platform_tag(self):
+    return subjects.str(
+        self.actual.platform_tag,
+        meta = self.meta.derive("platform_tag()"),
+    )
+
 def _py_cc_toolchain_info_subject_python_version(self):
     return subjects.str(
         self.actual.python_version,
         meta = self.meta.derive("python_version()"),
     )
 
+def _py_cc_toolchain_info_subject_sys_platform(self):
+    return subjects.str(
+        self.actual.sys_platform,
+        meta = self.meta.derive("sys_platform()"),
+    )
+
 # Disable this to aid doc generation
 # buildifier: disable=name-conventions
 PyCcToolchainInfoSubject = struct(