From c761deeb780549a37806fc185e53b307ccf0c7b4 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sat, 4 Apr 2026 19:46:46 -0700 Subject: [PATCH 1/9] feat: have zipapp main also use hash for extraction --- ...m_python_zipapp_external_bootstrap_test.sh | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/py_zipapp/system_python_zipapp_external_bootstrap_test.sh b/tests/py_zipapp/system_python_zipapp_external_bootstrap_test.sh index bb4ba640d3..87a27b421e 100755 --- a/tests/py_zipapp/system_python_zipapp_external_bootstrap_test.sh +++ b/tests/py_zipapp/system_python_zipapp_external_bootstrap_test.sh @@ -29,5 +29,30 @@ if [[ ! -d "$RULES_PYTHON_EXTRACT_ROOT" ]]; then exit 1 fi +# The extract dir is _main/tests/py_zipapp/system_python_zipapp +# The new structure should be $RULES_PYTHON_EXTRACT_ROOT/_main/tests/py_zipapp/system_python_zipapp//runfiles +# We check that there is a subdirectory under the expected extract dir. +EXTRACT_DIR="_main/tests/py_zipapp/system_python_zipapp" +if [[ ! -d "$RULES_PYTHON_EXTRACT_ROOT/$EXTRACT_DIR" ]]; then + echo "Error: Extract directory $RULES_PYTHON_EXTRACT_ROOT/$EXTRACT_DIR was not created!" + exit 1 +fi + +# Check for the extra hash component. +# Before the change, $RULES_PYTHON_EXTRACT_ROOT/$EXTRACT_DIR/runfiles would exist. +# After the change, $RULES_PYTHON_EXTRACT_ROOT/$EXTRACT_DIR//runfiles will exist. +# We look for any directory under $EXTRACT_DIR that contains 'runfiles'. +if ! find "$RULES_PYTHON_EXTRACT_ROOT/$EXTRACT_DIR" -maxdepth 2 -name runfiles | grep -q "runfiles"; then + echo "Error: Could not find 'runfiles' directory under $RULES_PYTHON_EXTRACT_ROOT/$EXTRACT_DIR" + exit 1 +fi + +# Specifically check that there's an intermediate directory between EXTRACT_DIR and runfiles +# find ... -mindepth 2 -maxdepth 2 -name runfiles should find it only if there is an intermediate dir. +if ! find "$RULES_PYTHON_EXTRACT_ROOT/$EXTRACT_DIR" -mindepth 2 -maxdepth 2 -name runfiles | grep -q "runfiles"; then + echo "Error: 'runfiles' directory is not at the expected depth. Missing APP_HASH component?" + exit 1 +fi + echo "Running zipapp with extract root set a second time..." "$PYTHON" "$ZIPAPP" From 42c39fe8b559f20ee6932e604783f95f2c4772c6 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sat, 4 Apr 2026 20:14:02 -0700 Subject: [PATCH 2/9] basic impl --- python/private/zipapp/py_zipapp_rule.bzl | 97 ++++++++++++++++++---- python/private/zipapp/zip_main_template.py | 3 +- tools/BUILD.bazel | 1 + tools/private/BUILD.bazel | 10 +++ tools/private/zipapp/BUILD.bazel | 20 +++++ tools/private/zipapp/zip_main_maker.py | 75 +++++++++++++++++ 6 files changed, 189 insertions(+), 17 deletions(-) create mode 100644 tools/private/zipapp/zip_main_maker.py diff --git a/python/private/zipapp/py_zipapp_rule.bzl b/python/private/zipapp/py_zipapp_rule.bzl index ac7944726f..b04fe3b68b 100644 --- a/python/private/zipapp/py_zipapp_rule.bzl +++ b/python/private/zipapp/py_zipapp_rule.bzl @@ -18,7 +18,45 @@ def _is_symlink(f): else: return "-1" -def _create_zipapp_main_py(ctx, py_runtime, py_executable, stage2_bootstrap): +def _map_zip_main_empty_filenames(list_paths_cb): + return list_paths_cb().to_list() + +def _map_zip_main_runfiles(file): + return file.short_path + "|" + file.path + +def _map_zip_main_symlinks(entry): + return entry.path + "|" + entry.target_file.path + +def _map_zip_main_root_symlinks(entry): + return entry.path + "|" + entry.target_file.path + +def _build_zip_main_hash_files_manifest(ctx, manifest, runfiles, inputs): + manifest.add_all( + # NOTE: Accessing runfiles.empty_filenames materializes them. A lambda + # is used to defer that. + [lambda: runfiles.empty_filenames], + map_each = _map_zip_main_empty_filenames, + allow_closure = True, + ) + + manifest.add_all(runfiles.files, map_each = _map_zip_main_runfiles) + manifest.add_all(runfiles.symlinks, map_each = _map_zip_main_symlinks) + manifest.add_all(runfiles.root_symlinks, map_each = _map_zip_main_root_symlinks) + + zip_repo_mapping_manifest = maybe_create_repo_mapping( + ctx = ctx, + runfiles = runfiles, + ) + if zip_repo_mapping_manifest: + # NOTE: rf-root-symlink is used to make it show up under the runfiles + # subdirectory within the zip. + manifest.add( + zip_repo_mapping_manifest.path, + format = "rf-root-symlink|0|_repo_mapping|%s", + ) + inputs.append(zip_repo_mapping_manifest) + +def _create_zipapp_main_py(ctx, py_runtime, py_executable, stage2_bootstrap, runfiles): venv_python_exe = py_executable.venv_python_exe if venv_python_exe: venv_python_exe_path = runfiles_root_path(ctx, venv_python_exe.short_path) @@ -31,20 +69,37 @@ def _create_zipapp_main_py(ctx, py_runtime, py_executable, stage2_bootstrap): python_binary_actual_path = py_runtime.interpreter_path zip_main_py = ctx.actions.declare_file(ctx.label.name + ".zip_main.py") - ctx.actions.expand_template( - template = py_runtime.zip_main_template, - output = zip_main_py, - substitutions = { - "%EXTRACT_DIR%": paths.join( - (ctx.label.repo_name or "_main"), - ctx.label.package, - ctx.label.name, - ), - "%python_binary%": venv_python_exe_path, - "%python_binary_actual%": python_binary_actual_path, - "%stage2_bootstrap%": runfiles_root_path(ctx, stage2_bootstrap.short_path), - "%workspace_name%": ctx.workspace_name, - }, + + args = ctx.actions.args() + args.add("--template", py_runtime.zip_main_template) + args.add("--output", zip_main_py) + + args.add( + "%EXTRACT_DIR%=" + paths.join( + (ctx.label.repo_name or "_main"), + ctx.label.package, + ctx.label.name, + ), + format = "--substitution=%s", + ) + args.add("%python_binary%=" + venv_python_exe_path, format = "--substitution=%s") + args.add("%python_binary_actual%=" + python_binary_actual_path, format = "--substitution=%s") + args.add("%stage2_bootstrap%=" + runfiles_root_path(ctx, stage2_bootstrap.short_path), format = "--substitution=%s") + args.add("%workspace_name%=" + ctx.workspace_name, format = "--substitution=%s") + + hash_files_manifest = ctx.actions.args() + hash_files_manifest.use_param_file("--hash_files_manifest=%s", use_always = True) + hash_files_manifest.set_param_file_format("multiline") + _build_zip_main_hash_files_manifest(ctx, runfiles, manifest, inputs) + + actions_run( + ctx, + executable = ctx.attr._zip_main_maker, + arguments = [args, hash_files_manifest], + inputs = depset([py_runtime.zip_main_template], transitive = [runfiles_files]), + outputs = [zip_main_py], + mnemonic = "PyZipAppCreateMainPy", + progress_message = "Generating zipapp __main__.py: %{label}", ) return zip_main_py @@ -106,7 +161,13 @@ def _create_zip(ctx, py_runtime, py_executable, stage2_bootstrap): runfiles = runfiles.build(ctx) - zip_main = _create_zipapp_main_py(ctx, py_runtime, py_executable, stage2_bootstrap) + zip_main = _create_zipapp_main_py( + ctx, + py_runtime, + py_executable, + stage2_bootstrap, + runfiles.build(ctx), + ) inputs = _build_manifest(ctx, manifest, runfiles, zip_main) zipper_args = ctx.actions.args() @@ -323,6 +384,10 @@ Whether the output should be an executable zip file. cfg = "exec", default = "//tools/private/zipapp:zipper", ), + "_zip_main_maker": attr.label( + cfg = "exec", + default = "//tools/private/zipapp:zip_main_maker", + ), } | ({ "_windows_launcher_maker": attr.label( default = "@bazel_tools//tools/launcher:launcher_maker", diff --git a/python/private/zipapp/zip_main_template.py b/python/private/zipapp/zip_main_template.py index 3c25d1d722..ca0ade34eb 100644 --- a/python/private/zipapp/zip_main_template.py +++ b/python/private/zipapp/zip_main_template.py @@ -39,6 +39,7 @@ _WORKSPACE_NAME = "%workspace_name%" # relative path under EXTRACT_ROOT to extract to. EXTRACT_DIR = "%EXTRACT_DIR%" +APP_HASH = "%APP_HASH%" EXTRACT_ROOT = os.environ.get("RULES_PYTHON_EXTRACT_ROOT") @@ -197,7 +198,7 @@ def extract_zip(zip_path, dest_dir): # Create the runfiles tree by extracting the zip file def create_runfiles_root(): if EXTRACT_ROOT: - extract_root = join(EXTRACT_ROOT, EXTRACT_DIR) + extract_root = join(EXTRACT_ROOT, EXTRACT_DIR, APP_HASH) else: extract_root = tempfile.mkdtemp("", "Bazel.runfiles_") extract_zip(dirname(__file__), extract_root) diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index 0fcce8f729..7829b33318 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -31,6 +31,7 @@ filegroup( "wheelmaker.py", "//tools/launcher:distribution", "//tools/precompiler:distribution", + "//tools/private:distribution", "//tools/publish:distribution", ], visibility = ["//:__pkg__"], diff --git a/tools/private/BUILD.bazel b/tools/private/BUILD.bazel index e69de29bb2..adc8de3b0f 100644 --- a/tools/private/BUILD.bazel +++ b/tools/private/BUILD.bazel @@ -0,0 +1,10 @@ +package( + default_visibility = ["//:__subpackages__"], +) + +filegroup( + name = "distribution", + srcs = glob(["**"]) + [ + "//tools/private/zipapp:distribution", + ], +) diff --git a/tools/private/zipapp/BUILD.bazel b/tools/private/zipapp/BUILD.bazel index 7a2002cd72..7420776ef3 100644 --- a/tools/private/zipapp/BUILD.bazel +++ b/tools/private/zipapp/BUILD.bazel @@ -34,3 +34,23 @@ py_library( name = "exe_zip_maker_lib", srcs = ["exe_zip_maker.py"], ) + +py_interpreter_program( + name = "zip_main_maker", + main = "zip_main_maker.py", + visibility = [ + # Not actually public. Only public so rules_python-generated toolchains + # are able to reference it. + "//visibility:public", + ], +) + +py_library( + name = "zip_main_maker_lib", + srcs = ["zip_main_maker.py"], +) + +filegroup( + name = "distribution", + srcs = glob(["**"]), +) diff --git a/tools/private/zipapp/zip_main_maker.py b/tools/private/zipapp/zip_main_maker.py new file mode 100644 index 0000000000..8e3fe276e5 --- /dev/null +++ b/tools/private/zipapp/zip_main_maker.py @@ -0,0 +1,75 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Creates the __main__.py for a zipapp by populating a template. + +This program also calculates a hash of the application files to include in +the template, which allows making the extraction directory unique to the +content of the zipapp. +""" + +import argparse +import hashlib +import os + + +def main(): + parser = argparse.ArgumentParser(fromfile_prefix_chars="@") + parser.add_argument("--template", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--substitution", action="append", default=[]) + parser.add_argument("hash_inputs", nargs="*") + args = parser.parse_args() + + # We want the hash to be deterministic. + # The order of files matters for the hash. + # We hash both the short path (to capture structure) and the content. + # Wait, we only have the full path here. + # Bazel provides full paths. + + h = hashlib.sha256() + for path in sorted(args.hash_inputs): + # We don't have the 'short_path' here easily unless we pass it. + # But for the purpose of a unique hash, the full path is probably fine + # as long as it's stable within a build. + # However, full paths in Bazel can contain 'bazel-out/k8-fastbuild/bin/...'. + # That's still stable for a given configuration. + h.update(path.encode("utf-8")) + if os.path.isfile(path): + with open(path, "rb") as f: + while True: + chunk = f.read(65536) + if not chunk: + break + h.update(chunk) + + app_hash = h.hexdigest() + + substitutions = {"%APP_HASH%": app_hash} + for s in args.substitution: + key, val = s.split("=", 1) + substitutions[key] = val + + with open(args.template, "r", encoding="utf-8") as f: + content = f.read() + + for key, val in substitutions.items(): + content = content.replace(key, val) + + with open(args.output, "w", encoding="utf-8") as f: + f.write(content) + + +if __name__ == "__main__": + main() From 92c8f5d9b233d016267a38e8d01709ac7d2064ba Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sat, 4 Apr 2026 23:17:25 -0700 Subject: [PATCH 3/9] handle new manifest format --- python/private/zipapp/py_zipapp_rule.bzl | 8 +- tools/private/zipapp/BUILD.bazel | 7 ++ tools/private/zipapp/zip_main_maker.py | 18 ++--- tools/private/zipapp/zip_main_maker_test.py | 81 +++++++++++++++++++++ 4 files changed, 102 insertions(+), 12 deletions(-) create mode 100644 tools/private/zipapp/zip_main_maker_test.py diff --git a/python/private/zipapp/py_zipapp_rule.bzl b/python/private/zipapp/py_zipapp_rule.bzl index b04fe3b68b..092665e96e 100644 --- a/python/private/zipapp/py_zipapp_rule.bzl +++ b/python/private/zipapp/py_zipapp_rule.bzl @@ -90,13 +90,15 @@ def _create_zipapp_main_py(ctx, py_runtime, py_executable, stage2_bootstrap, run hash_files_manifest = ctx.actions.args() hash_files_manifest.use_param_file("--hash_files_manifest=%s", use_always = True) hash_files_manifest.set_param_file_format("multiline") - _build_zip_main_hash_files_manifest(ctx, runfiles, manifest, inputs) + + inputs = [py_runtime.zip_main_template] + _build_zip_main_hash_files_manifest(ctx, hash_files_manifest, runfiles, inputs) actions_run( ctx, executable = ctx.attr._zip_main_maker, arguments = [args, hash_files_manifest], - inputs = depset([py_runtime.zip_main_template], transitive = [runfiles_files]), + inputs = depset(inputs, transitive = [runfiles.files]), outputs = [zip_main_py], mnemonic = "PyZipAppCreateMainPy", progress_message = "Generating zipapp __main__.py: %{label}", @@ -166,7 +168,7 @@ def _create_zip(ctx, py_runtime, py_executable, stage2_bootstrap): py_runtime, py_executable, stage2_bootstrap, - runfiles.build(ctx), + runfiles, ) inputs = _build_manifest(ctx, manifest, runfiles, zip_main) diff --git a/tools/private/zipapp/BUILD.bazel b/tools/private/zipapp/BUILD.bazel index 7420776ef3..bd274d7fb7 100644 --- a/tools/private/zipapp/BUILD.bazel +++ b/tools/private/zipapp/BUILD.bazel @@ -1,4 +1,5 @@ load("//python:py_library.bzl", "py_library") +load("//python:py_test.bzl", "py_test") load("//python/private:py_interpreter_program.bzl", "py_interpreter_program") # buildifier: disable=bzl-visibility package( @@ -50,6 +51,12 @@ py_library( srcs = ["zip_main_maker.py"], ) +py_test( + name = "zip_main_maker_test", + srcs = ["zip_main_maker_test.py"], + deps = [":zip_main_maker_lib"], +) + filegroup( name = "distribution", srcs = glob(["**"]), diff --git a/tools/private/zipapp/zip_main_maker.py b/tools/private/zipapp/zip_main_maker.py index 8e3fe276e5..dc66a70870 100644 --- a/tools/private/zipapp/zip_main_maker.py +++ b/tools/private/zipapp/zip_main_maker.py @@ -29,7 +29,7 @@ def main(): parser.add_argument("--template", required=True) parser.add_argument("--output", required=True) parser.add_argument("--substitution", action="append", default=[]) - parser.add_argument("hash_inputs", nargs="*") + parser.add_argument("--hash_files_manifest", required=True) args = parser.parse_args() # We want the hash to be deterministic. @@ -39,14 +39,14 @@ def main(): # Bazel provides full paths. h = hashlib.sha256() - for path in sorted(args.hash_inputs): - # We don't have the 'short_path' here easily unless we pass it. - # But for the purpose of a unique hash, the full path is probably fine - # as long as it's stable within a build. - # However, full paths in Bazel can contain 'bazel-out/k8-fastbuild/bin/...'. - # That's still stable for a given configuration. - h.update(path.encode("utf-8")) - if os.path.isfile(path): + with open(args.hash_files_manifest, "r", encoding="utf-8") as f: + manifest_lines = f.read().splitlines() + + for line in sorted(manifest_lines): + h.update(line.encode("utf-8")) + parts = line.split("|") + path = parts[-1] + if path and os.path.isfile(path): with open(path, "rb") as f: while True: chunk = f.read(65536) diff --git a/tools/private/zipapp/zip_main_maker_test.py b/tools/private/zipapp/zip_main_maker_test.py new file mode 100644 index 0000000000..8b377539e7 --- /dev/null +++ b/tools/private/zipapp/zip_main_maker_test.py @@ -0,0 +1,81 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import os +import tempfile +import unittest +from unittest import mock + +from tools.private.zipapp import zip_main_maker + +class ZipMainMakerTest(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + self.addCleanup(self.temp_dir.cleanup) + + def test_creates_zip_main(self): + template_path = os.path.join(self.temp_dir.name, "template.py") + with open(template_path, "w", encoding="utf-8") as f: + f.write("hash=%APP_HASH%\nfoo=%FOO%\n") + + output_path = os.path.join(self.temp_dir.name, "output.py") + + file1_path = os.path.join(self.temp_dir.name, "file1.txt") + with open(file1_path, "wb") as f: + f.write(b"content1") + + file2_path = os.path.join(self.temp_dir.name, "file2.txt") + with open(file2_path, "wb") as f: + f.write(b"content2") + + manifest_path = os.path.join(self.temp_dir.name, "manifest.txt") + with open(manifest_path, "w", encoding="utf-8") as f: + f.write(f"rf-file|1|file1.txt|{file1_path}\n") + f.write(f"rf-file|0|file2.txt|{file2_path}\n") + + argv = [ + "zip_main_maker.py", + "--template", template_path, + "--output", output_path, + "--substitution", "%FOO%=bar", + "--hash_files_manifest", manifest_path, + ] + + with mock.patch("sys.argv", argv): + zip_main_maker.main() + + # Calculate expected hash + h = hashlib.sha256() + line1 = f"rf-file|1|file1.txt|{file1_path}" + line2 = f"rf-file|0|file2.txt|{file2_path}" + + # Sort lines like the program does + lines = sorted([line1, line2]) + for line in lines: + h.update(line.encode("utf-8")) + parts = line.split("|") + path = parts[-1] + with open(path, "rb") as f: + h.update(f.read()) + + expected_hash = h.hexdigest() + + with open(output_path, "r", encoding="utf-8") as f: + content = f.read() + + self.assertEqual(content, f"hash={expected_hash}\nfoo=bar\n") + +if __name__ == "__main__": + unittest.main() From 6eceba0adb706b1e76fade8a08e0d63a6c361852 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sat, 4 Apr 2026 23:21:13 -0700 Subject: [PATCH 4/9] move test --- tests/tools/zipapp/BUILD.bazel | 6 ++++++ .../tools}/zipapp/zip_main_maker_test.py | 0 tools/private/zipapp/BUILD.bazel | 9 +-------- 3 files changed, 7 insertions(+), 8 deletions(-) rename {tools/private => tests/tools}/zipapp/zip_main_maker_test.py (100%) diff --git a/tests/tools/zipapp/BUILD.bazel b/tests/tools/zipapp/BUILD.bazel index 84902b76b8..b71e9b2589 100644 --- a/tests/tools/zipapp/BUILD.bazel +++ b/tests/tools/zipapp/BUILD.bazel @@ -11,3 +11,9 @@ py_test( srcs = ["exe_zip_maker_test.py"], deps = ["//tools/private/zipapp:exe_zip_maker_lib"], ) + +py_test( + name = "zip_main_maker_test", + srcs = ["zip_main_maker_test.py"], + deps = ["//tools/private/zipapp:zip_main_maker_lib"], +) diff --git a/tools/private/zipapp/zip_main_maker_test.py b/tests/tools/zipapp/zip_main_maker_test.py similarity index 100% rename from tools/private/zipapp/zip_main_maker_test.py rename to tests/tools/zipapp/zip_main_maker_test.py diff --git a/tools/private/zipapp/BUILD.bazel b/tools/private/zipapp/BUILD.bazel index bd274d7fb7..40e2f8e85f 100644 --- a/tools/private/zipapp/BUILD.bazel +++ b/tools/private/zipapp/BUILD.bazel @@ -1,5 +1,4 @@ load("//python:py_library.bzl", "py_library") -load("//python:py_test.bzl", "py_test") load("//python/private:py_interpreter_program.bzl", "py_interpreter_program") # buildifier: disable=bzl-visibility package( @@ -51,13 +50,7 @@ py_library( srcs = ["zip_main_maker.py"], ) -py_test( - name = "zip_main_maker_test", - srcs = ["zip_main_maker_test.py"], - deps = [":zip_main_maker_lib"], -) - filegroup( name = "distribution", srcs = glob(["**"]), -) +) \ No newline at end of file From e4cda7f8d697bc693e4806d0165ee54cc12f7258 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sat, 4 Apr 2026 23:33:26 -0700 Subject: [PATCH 5/9] cleanup, refactoring --- python/private/zipapp/py_zipapp_rule.bzl | 28 +++--- ...m_python_zipapp_external_bootstrap_test.sh | 14 +-- tests/tools/zipapp/zip_main_maker_test.py | 69 +++++++------- tools/private/zipapp/BUILD.bazel | 2 +- tools/private/zipapp/zip_main_maker.py | 90 +++++++++++-------- 5 files changed, 103 insertions(+), 100 deletions(-) diff --git a/python/private/zipapp/py_zipapp_rule.bzl b/python/private/zipapp/py_zipapp_rule.bzl index 092665e96e..3a8b2a59d3 100644 --- a/python/private/zipapp/py_zipapp_rule.bzl +++ b/python/private/zipapp/py_zipapp_rule.bzl @@ -43,18 +43,19 @@ def _build_zip_main_hash_files_manifest(ctx, manifest, runfiles, inputs): manifest.add_all(runfiles.symlinks, map_each = _map_zip_main_symlinks) manifest.add_all(runfiles.root_symlinks, map_each = _map_zip_main_root_symlinks) + inputs.add(runfiles.files) + inputs.add([entry.target_file for entry in runfiles.symlinks.to_list()]) + inputs.add([entry.target_file for entry in runfiles.root_symlinks.to_list()]) + zip_repo_mapping_manifest = maybe_create_repo_mapping( ctx = ctx, runfiles = runfiles, ) if zip_repo_mapping_manifest: - # NOTE: rf-root-symlink is used to make it show up under the runfiles - # subdirectory within the zip. manifest.add( - zip_repo_mapping_manifest.path, - format = "rf-root-symlink|0|_repo_mapping|%s", + "_repo_mapping|" + zip_repo_mapping_manifest.path, ) - inputs.append(zip_repo_mapping_manifest) + inputs.add(zip_repo_mapping_manifest) def _create_zipapp_main_py(ctx, py_runtime, py_executable, stage2_bootstrap, runfiles): venv_python_exe = py_executable.venv_python_exe @@ -71,8 +72,8 @@ def _create_zipapp_main_py(ctx, py_runtime, py_executable, stage2_bootstrap, run zip_main_py = ctx.actions.declare_file(ctx.label.name + ".zip_main.py") args = ctx.actions.args() - args.add("--template", py_runtime.zip_main_template) - args.add("--output", zip_main_py) + args.add(py_runtime.zip_main_template, format = "--template=%s") + args.add(zip_main_py, format = "--output=%s") args.add( "%EXTRACT_DIR%=" + paths.join( @@ -91,14 +92,15 @@ def _create_zipapp_main_py(ctx, py_runtime, py_executable, stage2_bootstrap, run hash_files_manifest.use_param_file("--hash_files_manifest=%s", use_always = True) hash_files_manifest.set_param_file_format("multiline") - inputs = [py_runtime.zip_main_template] + inputs = builders.DepsetBuilder() + inputs.add(py_runtime.zip_main_template) _build_zip_main_hash_files_manifest(ctx, hash_files_manifest, runfiles, inputs) actions_run( ctx, executable = ctx.attr._zip_main_maker, arguments = [args, hash_files_manifest], - inputs = depset(inputs, transitive = [runfiles.files]), + inputs = inputs.build(), outputs = [zip_main_py], mnemonic = "PyZipAppCreateMainPy", progress_message = "Generating zipapp __main__.py: %{label}", @@ -378,6 +380,10 @@ Whether the output should be an executable zip file. "@platforms//os:windows", ], ), + "_zip_main_maker": attr.label( + cfg = "exec", + default = "//tools/private/zipapp:zip_main_maker", + ), "_zip_shell_template": attr.label( default = ":zip_shell_template", allow_single_file = True, @@ -386,10 +392,6 @@ Whether the output should be an executable zip file. cfg = "exec", default = "//tools/private/zipapp:zipper", ), - "_zip_main_maker": attr.label( - cfg = "exec", - default = "//tools/private/zipapp:zip_main_maker", - ), } | ({ "_windows_launcher_maker": attr.label( default = "@bazel_tools//tools/launcher:launcher_maker", diff --git a/tests/py_zipapp/system_python_zipapp_external_bootstrap_test.sh b/tests/py_zipapp/system_python_zipapp_external_bootstrap_test.sh index 87a27b421e..ed09b02ba7 100755 --- a/tests/py_zipapp/system_python_zipapp_external_bootstrap_test.sh +++ b/tests/py_zipapp/system_python_zipapp_external_bootstrap_test.sh @@ -41,16 +41,10 @@ fi # Check for the extra hash component. # Before the change, $RULES_PYTHON_EXTRACT_ROOT/$EXTRACT_DIR/runfiles would exist. # After the change, $RULES_PYTHON_EXTRACT_ROOT/$EXTRACT_DIR//runfiles will exist. -# We look for any directory under $EXTRACT_DIR that contains 'runfiles'. -if ! find "$RULES_PYTHON_EXTRACT_ROOT/$EXTRACT_DIR" -maxdepth 2 -name runfiles | grep -q "runfiles"; then - echo "Error: Could not find 'runfiles' directory under $RULES_PYTHON_EXTRACT_ROOT/$EXTRACT_DIR" - exit 1 -fi - -# Specifically check that there's an intermediate directory between EXTRACT_DIR and runfiles -# find ... -mindepth 2 -maxdepth 2 -name runfiles should find it only if there is an intermediate dir. -if ! find "$RULES_PYTHON_EXTRACT_ROOT/$EXTRACT_DIR" -mindepth 2 -maxdepth 2 -name runfiles | grep -q "runfiles"; then - echo "Error: 'runfiles' directory is not at the expected depth. Missing APP_HASH component?" +# We use glob expansion to check for the expected depth. +# Note: [ -d ... ] expands globs, while [[ -d ... ]] does not. +if [ ! -d "$RULES_PYTHON_EXTRACT_ROOT/$EXTRACT_DIR"/*/runfiles ]; then + echo "Error: Could not find 'runfiles' directory at expected depth $RULES_PYTHON_EXTRACT_ROOT/$EXTRACT_DIR/*/runfiles" exit 1 fi diff --git a/tests/tools/zipapp/zip_main_maker_test.py b/tests/tools/zipapp/zip_main_maker_test.py index 8b377539e7..2d53ccf72d 100644 --- a/tests/tools/zipapp/zip_main_maker_test.py +++ b/tests/tools/zipapp/zip_main_maker_test.py @@ -1,17 +1,3 @@ -# Copyright 2024 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - import hashlib import os import tempfile @@ -20,6 +6,7 @@ from tools.private.zipapp import zip_main_maker + class ZipMainMakerTest(unittest.TestCase): def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -29,53 +16,61 @@ def test_creates_zip_main(self): template_path = os.path.join(self.temp_dir.name, "template.py") with open(template_path, "w", encoding="utf-8") as f: f.write("hash=%APP_HASH%\nfoo=%FOO%\n") - + output_path = os.path.join(self.temp_dir.name, "output.py") - + file1_path = os.path.join(self.temp_dir.name, "file1.txt") with open(file1_path, "wb") as f: f.write(b"content1") - + file2_path = os.path.join(self.temp_dir.name, "file2.txt") with open(file2_path, "wb") as f: f.write(b"content2") - + manifest_path = os.path.join(self.temp_dir.name, "manifest.txt") with open(manifest_path, "w", encoding="utf-8") as f: - f.write(f"rf-file|1|file1.txt|{file1_path}\n") - f.write(f"rf-file|0|file2.txt|{file2_path}\n") - + f.write(f"file1.txt|{file1_path}\n") + f.write(f"file2.txt|{file2_path}\n") + f.write(f"empty_file.txt\n") + argv = [ "zip_main_maker.py", - "--template", template_path, - "--output", output_path, - "--substitution", "%FOO%=bar", - "--hash_files_manifest", manifest_path, + "--template", + template_path, + "--output", + output_path, + "--substitution", + "%FOO%=bar", + "--hash_files_manifest", + manifest_path, ] - + with mock.patch("sys.argv", argv): zip_main_maker.main() - + # Calculate expected hash h = hashlib.sha256() - line1 = f"rf-file|1|file1.txt|{file1_path}" - line2 = f"rf-file|0|file2.txt|{file2_path}" - + line1 = f"file1.txt|{file1_path}" + line2 = f"file2.txt|{file2_path}" + line3 = f"empty_file.txt" + # Sort lines like the program does - lines = sorted([line1, line2]) + lines = sorted([line1, line2, line3]) for line in lines: h.update(line.encode("utf-8")) parts = line.split("|") - path = parts[-1] - with open(path, "rb") as f: - h.update(f.read()) - + if len(parts) == 2: + path = parts[1] + with open(path, "rb") as f: + h.update(f.read()) + expected_hash = h.hexdigest() - + with open(output_path, "r", encoding="utf-8") as f: content = f.read() - + self.assertEqual(content, f"hash={expected_hash}\nfoo=bar\n") + if __name__ == "__main__": unittest.main() diff --git a/tools/private/zipapp/BUILD.bazel b/tools/private/zipapp/BUILD.bazel index 40e2f8e85f..7420776ef3 100644 --- a/tools/private/zipapp/BUILD.bazel +++ b/tools/private/zipapp/BUILD.bazel @@ -53,4 +53,4 @@ py_library( filegroup( name = "distribution", srcs = glob(["**"]), -) \ No newline at end of file +) diff --git a/tools/private/zipapp/zip_main_maker.py b/tools/private/zipapp/zip_main_maker.py index dc66a70870..00543d5ebe 100644 --- a/tools/private/zipapp/zip_main_maker.py +++ b/tools/private/zipapp/zip_main_maker.py @@ -1,17 +1,3 @@ -# Copyright 2024 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - """Creates the __main__.py for a zipapp by populating a template. This program also calculates a hash of the application files to include in @@ -23,53 +9,79 @@ import hashlib import os +BLOCK_SIZE = 256 * 1024 -def main(): + +def create_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(fromfile_prefix_chars="@") parser.add_argument("--template", required=True) parser.add_argument("--output", required=True) parser.add_argument("--substitution", action="append", default=[]) - parser.add_argument("--hash_files_manifest", required=True) - args = parser.parse_args() + parser.add_argument( + "--hash_files_manifest", + required=True, + help="A file containing lines of either 'empty_file_path' or 'short_path|readable_path'", + ) + return parser + + +def compute_inputs_hash(manifest_path: str) -> str: - # We want the hash to be deterministic. - # The order of files matters for the hash. - # We hash both the short path (to capture structure) and the content. - # Wait, we only have the full path here. - # Bazel provides full paths. - h = hashlib.sha256() - with open(args.hash_files_manifest, "r", encoding="utf-8") as f: + with open(manifest_path, "r", encoding="utf-8") as f: manifest_lines = f.read().splitlines() + # Sort lines for determinism. Hash the paths (to capture structure) and the + # content. for line in sorted(manifest_lines): h.update(line.encode("utf-8")) parts = line.split("|") - path = parts[-1] - if path and os.path.isfile(path): - with open(path, "rb") as f: - while True: - chunk = f.read(65536) - if not chunk: - break - h.update(chunk) + if len(parts) != 2: + # If there's no '|', it's just an empty file path, which has + # already been added to the hash. + continue + path = parts[1] - app_hash = h.hexdigest() + # We hash the content for regular files. For symlinks, we hash the + # target path. This is more robust in a sandbox where symlink targets + # might not be present if they are absolute paths or outside the + # runfiles. + if os.path.islink(path): + h.update(os.readlink(path).encode("utf-8")) + with open(path, "rb") as f: + while True: + chunk = f.read(BLOCK_SIZE) + if not chunk: + break + h.update(chunk) + + return h.hexdigest() - substitutions = {"%APP_HASH%": app_hash} - for s in args.substitution: - key, val = s.split("=", 1) - substitutions[key] = val - with open(args.template, "r", encoding="utf-8") as f: +def expand_template(template_path: str, output_path: str, substitutions: dict) -> None: + with open(template_path, "r", encoding="utf-8") as f: content = f.read() for key, val in substitutions.items(): content = content.replace(key, val) - with open(args.output, "w", encoding="utf-8") as f: + with open(output_path, "w", encoding="utf-8") as f: f.write(content) +def main(): + parser = create_parser() + args = parser.parse_args() + + app_hash = compute_inputs_hash(args.hash_files_manifest) + + substitutions = {"%APP_HASH%": app_hash} + for s in args.substitution: + key, val = s.split("=", 1) + substitutions[key] = val + + expand_template(args.template, args.output, substitutions) + + if __name__ == "__main__": main() From 8f190694a5cd1616ea86b2188f8954176eb8627c Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 5 Apr 2026 00:07:33 -0700 Subject: [PATCH 6/9] use rf-XXX manifest format to handle symlinks correctly --- python/private/zipapp/py_zipapp_rule.bzl | 22 ++------- ...m_python_zipapp_external_bootstrap_test.sh | 2 - tests/tools/zipapp/zip_main_maker_test.py | 49 ++++++++++++++----- tools/private/zipapp/zip_main_maker.py | 41 +++++++++------- 4 files changed, 64 insertions(+), 50 deletions(-) diff --git a/python/private/zipapp/py_zipapp_rule.bzl b/python/private/zipapp/py_zipapp_rule.bzl index 3a8b2a59d3..e50a48b339 100644 --- a/python/private/zipapp/py_zipapp_rule.bzl +++ b/python/private/zipapp/py_zipapp_rule.bzl @@ -18,30 +18,18 @@ def _is_symlink(f): else: return "-1" -def _map_zip_main_empty_filenames(list_paths_cb): - return list_paths_cb().to_list() - -def _map_zip_main_runfiles(file): - return file.short_path + "|" + file.path - -def _map_zip_main_symlinks(entry): - return entry.path + "|" + entry.target_file.path - -def _map_zip_main_root_symlinks(entry): - return entry.path + "|" + entry.target_file.path - def _build_zip_main_hash_files_manifest(ctx, manifest, runfiles, inputs): manifest.add_all( # NOTE: Accessing runfiles.empty_filenames materializes them. A lambda # is used to defer that. [lambda: runfiles.empty_filenames], - map_each = _map_zip_main_empty_filenames, + map_each = _map_zip_empty_filenames, allow_closure = True, ) - manifest.add_all(runfiles.files, map_each = _map_zip_main_runfiles) - manifest.add_all(runfiles.symlinks, map_each = _map_zip_main_symlinks) - manifest.add_all(runfiles.root_symlinks, map_each = _map_zip_main_root_symlinks) + manifest.add_all(runfiles.files, map_each = _map_zip_runfiles) + manifest.add_all(runfiles.symlinks, map_each = _map_zip_symlinks) + manifest.add_all(runfiles.root_symlinks, map_each = _map_zip_root_symlinks) inputs.add(runfiles.files) inputs.add([entry.target_file for entry in runfiles.symlinks.to_list()]) @@ -53,7 +41,7 @@ def _build_zip_main_hash_files_manifest(ctx, manifest, runfiles, inputs): ) if zip_repo_mapping_manifest: manifest.add( - "_repo_mapping|" + zip_repo_mapping_manifest.path, + "rf-root-symlink|0|_repo_mapping|" + zip_repo_mapping_manifest.path, ) inputs.add(zip_repo_mapping_manifest) diff --git a/tests/py_zipapp/system_python_zipapp_external_bootstrap_test.sh b/tests/py_zipapp/system_python_zipapp_external_bootstrap_test.sh index ed09b02ba7..0c9ebe131d 100755 --- a/tests/py_zipapp/system_python_zipapp_external_bootstrap_test.sh +++ b/tests/py_zipapp/system_python_zipapp_external_bootstrap_test.sh @@ -39,8 +39,6 @@ if [[ ! -d "$RULES_PYTHON_EXTRACT_ROOT/$EXTRACT_DIR" ]]; then fi # Check for the extra hash component. -# Before the change, $RULES_PYTHON_EXTRACT_ROOT/$EXTRACT_DIR/runfiles would exist. -# After the change, $RULES_PYTHON_EXTRACT_ROOT/$EXTRACT_DIR//runfiles will exist. # We use glob expansion to check for the expected depth. # Note: [ -d ... ] expands globs, while [[ -d ... ]] does not. if [ ! -d "$RULES_PYTHON_EXTRACT_ROOT/$EXTRACT_DIR"/*/runfiles ]; then diff --git a/tests/tools/zipapp/zip_main_maker_test.py b/tests/tools/zipapp/zip_main_maker_test.py index 2d53ccf72d..afcaf294d1 100644 --- a/tests/tools/zipapp/zip_main_maker_test.py +++ b/tests/tools/zipapp/zip_main_maker_test.py @@ -27,11 +27,16 @@ def test_creates_zip_main(self): with open(file2_path, "wb") as f: f.write(b"content2") + # Add a symlink to test symlink hashing + symlink_path = os.path.join(self.temp_dir.name, "symlink.txt") + os.symlink(file1_path, symlink_path) + manifest_path = os.path.join(self.temp_dir.name, "manifest.txt") with open(manifest_path, "w", encoding="utf-8") as f: - f.write(f"file1.txt|{file1_path}\n") - f.write(f"file2.txt|{file2_path}\n") - f.write(f"empty_file.txt\n") + f.write(f"rf-file|0|file1.txt|{file1_path}\n") + f.write(f"rf-file|0|file2.txt|{file2_path}\n") + f.write(f"rf-symlink|1|symlink.txt|{symlink_path}\n") + f.write(f"rf-empty|empty_file.txt\n") argv = [ "zip_main_maker.py", @@ -50,19 +55,39 @@ def test_creates_zip_main(self): # Calculate expected hash h = hashlib.sha256() - line1 = f"file1.txt|{file1_path}" - line2 = f"file2.txt|{file2_path}" - line3 = f"empty_file.txt" + line1 = f"rf-file|0|file1.txt|{file1_path}" + line2 = f"rf-file|0|file2.txt|{file2_path}" + line3 = f"rf-symlink|1|symlink.txt|{symlink_path}" + line4 = f"rf-empty|empty_file.txt" # Sort lines like the program does - lines = sorted([line1, line2, line3]) + lines = sorted([line1, line2, line3, line4]) for line in lines: - h.update(line.encode("utf-8")) parts = line.split("|") - if len(parts) == 2: - path = parts[1] - with open(path, "rb") as f: - h.update(f.read()) + if len(parts) > 1: + _, rest = line.split("|", 1) + h.update(rest.encode("utf-8")) + else: + h.update(line.encode("utf-8")) + + type_ = parts[0] + if type_ == "rf-empty": + continue + if len(parts) >= 4: + is_symlink_str = parts[1] + path = parts[-1] + if not path: + continue + if is_symlink_str == "-1": + is_symlink = not os.path.exists(path) + else: + is_symlink = is_symlink_str == "1" + + if is_symlink: + h.update(os.readlink(path).encode("utf-8")) + else: + with open(path, "rb") as f: + h.update(f.read()) expected_hash = h.hexdigest() diff --git a/tools/private/zipapp/zip_main_maker.py b/tools/private/zipapp/zip_main_maker.py index 00543d5ebe..78ac17eb17 100644 --- a/tools/private/zipapp/zip_main_maker.py +++ b/tools/private/zipapp/zip_main_maker.py @@ -20,13 +20,12 @@ def create_parser() -> argparse.ArgumentParser: parser.add_argument( "--hash_files_manifest", required=True, - help="A file containing lines of either 'empty_file_path' or 'short_path|readable_path'", + help="A file containing lines in rf-XXX formats (rf-empty, rf-file, rf-symlink, etc.)", ) return parser def compute_inputs_hash(manifest_path: str) -> str: - h = hashlib.sha256() with open(manifest_path, "r", encoding="utf-8") as f: manifest_lines = f.read().splitlines() @@ -34,26 +33,30 @@ def compute_inputs_hash(manifest_path: str) -> str: # Sort lines for determinism. Hash the paths (to capture structure) and the # content. for line in sorted(manifest_lines): - h.update(line.encode("utf-8")) - parts = line.split("|") - if len(parts) != 2: - # If there's no '|', it's just an empty file path, which has - # already been added to the hash. + type_, _, rest = line.partition("|") + h.update(rest.encode("utf-8")) + parts = rest.split("|") + + if type_ == "rf-empty": continue - path = parts[1] - # We hash the content for regular files. For symlinks, we hash the - # target path. This is more robust in a sandbox where symlink targets - # might not be present if they are absolute paths or outside the - # runfiles. - if os.path.islink(path): + is_symlink_str = parts[0] + path = parts[-1] + + if is_symlink_str == "-1": + is_symlink = not os.path.exists(path) + else: + is_symlink = is_symlink_str == "1" + + if is_symlink: h.update(os.readlink(path).encode("utf-8")) - with open(path, "rb") as f: - while True: - chunk = f.read(BLOCK_SIZE) - if not chunk: - break - h.update(chunk) + else: + with open(path, "rb") as f: + while True: + chunk = f.read(BLOCK_SIZE) + if not chunk: + break + h.update(chunk) return h.hexdigest() From d05ee87cf55e51b53b9091eecbad56ced8057dcb Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 5 Apr 2026 00:33:04 -0700 Subject: [PATCH 7/9] reuse build manifest function --- python/private/zipapp/py_zipapp_rule.bzl | 48 ++++++------------------ 1 file changed, 12 insertions(+), 36 deletions(-) diff --git a/python/private/zipapp/py_zipapp_rule.bzl b/python/private/zipapp/py_zipapp_rule.bzl index e50a48b339..1655f63cc9 100644 --- a/python/private/zipapp/py_zipapp_rule.bzl +++ b/python/private/zipapp/py_zipapp_rule.bzl @@ -18,33 +18,6 @@ def _is_symlink(f): else: return "-1" -def _build_zip_main_hash_files_manifest(ctx, manifest, runfiles, inputs): - manifest.add_all( - # NOTE: Accessing runfiles.empty_filenames materializes them. A lambda - # is used to defer that. - [lambda: runfiles.empty_filenames], - map_each = _map_zip_empty_filenames, - allow_closure = True, - ) - - manifest.add_all(runfiles.files, map_each = _map_zip_runfiles) - manifest.add_all(runfiles.symlinks, map_each = _map_zip_symlinks) - manifest.add_all(runfiles.root_symlinks, map_each = _map_zip_root_symlinks) - - inputs.add(runfiles.files) - inputs.add([entry.target_file for entry in runfiles.symlinks.to_list()]) - inputs.add([entry.target_file for entry in runfiles.root_symlinks.to_list()]) - - zip_repo_mapping_manifest = maybe_create_repo_mapping( - ctx = ctx, - runfiles = runfiles, - ) - if zip_repo_mapping_manifest: - manifest.add( - "rf-root-symlink|0|_repo_mapping|" + zip_repo_mapping_manifest.path, - ) - inputs.add(zip_repo_mapping_manifest) - def _create_zipapp_main_py(ctx, py_runtime, py_executable, stage2_bootstrap, runfiles): venv_python_exe = py_executable.venv_python_exe if venv_python_exe: @@ -82,7 +55,7 @@ def _create_zipapp_main_py(ctx, py_runtime, py_executable, stage2_bootstrap, run inputs = builders.DepsetBuilder() inputs.add(py_runtime.zip_main_template) - _build_zip_main_hash_files_manifest(ctx, hash_files_manifest, runfiles, inputs) + _build_manifest(ctx, hash_files_manifest, runfiles, inputs) actions_run( ctx, @@ -107,9 +80,7 @@ def _map_zip_symlinks(entry): def _map_zip_root_symlinks(entry): return "rf-root-symlink|" + _is_symlink(entry.target_file) + "|" + entry.path + "|" + entry.target_file.path -def _build_manifest(ctx, manifest, runfiles, zip_main): - manifest.add("regular|0|__main__.py|{}".format(zip_main.path)) - +def _build_manifest(ctx, manifest, runfiles, inputs): manifest.add_all( # NOTE: Accessing runfiles.empty_filenames materializes them. A lambda # is used to defer that. @@ -122,7 +93,10 @@ def _build_manifest(ctx, manifest, runfiles, zip_main): manifest.add_all(runfiles.symlinks, map_each = _map_zip_symlinks) manifest.add_all(runfiles.root_symlinks, map_each = _map_zip_root_symlinks) - inputs = [zip_main] + inputs.add(runfiles.files) + inputs.add([entry.target_file for entry in runfiles.symlinks.to_list()]) + inputs.add([entry.target_file for entry in runfiles.root_symlinks.to_list()]) + zip_repo_mapping_manifest = maybe_create_repo_mapping( ctx = ctx, runfiles = runfiles, @@ -134,8 +108,7 @@ def _build_manifest(ctx, manifest, runfiles, zip_main): zip_repo_mapping_manifest.path, format = "rf-root-symlink|0|_repo_mapping|%s", ) - inputs.append(zip_repo_mapping_manifest) - return inputs + inputs.add(zip_repo_mapping_manifest) def _create_zip(ctx, py_runtime, py_executable, stage2_bootstrap): output = ctx.actions.declare_file(ctx.label.name + ".zip") @@ -160,7 +133,10 @@ def _create_zip(ctx, py_runtime, py_executable, stage2_bootstrap): stage2_bootstrap, runfiles, ) - inputs = _build_manifest(ctx, manifest, runfiles, zip_main) + inputs = builders.DepsetBuilder() + manifest.add("regular|0|__main__.py|{}".format(zip_main.path)) + inputs.add(zip_main) + _build_manifest(ctx, manifest, runfiles, inputs) zipper_args = ctx.actions.args() zipper_args.add(output) @@ -177,7 +153,7 @@ def _create_zip(ctx, py_runtime, py_executable, stage2_bootstrap): ctx, executable = ctx.attr._zipper, arguments = [manifest, zipper_args], - inputs = depset(inputs, transitive = [runfiles.files]), + inputs = inputs.build(), outputs = [output], mnemonic = "PyZipAppCreateZip", progress_message = "Reticulating zipapp archive: %{label} into %{output}", From 2c59a5bc148f42bb1eb3189e91aed8fae0313bfb Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 5 Apr 2026 08:52:56 -0700 Subject: [PATCH 8/9] add some progress logic to test --- .../system_python_zipapp_external_bootstrap_test.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/py_zipapp/system_python_zipapp_external_bootstrap_test.sh b/tests/py_zipapp/system_python_zipapp_external_bootstrap_test.sh index 0c9ebe131d..cc5694285f 100755 --- a/tests/py_zipapp/system_python_zipapp_external_bootstrap_test.sh +++ b/tests/py_zipapp/system_python_zipapp_external_bootstrap_test.sh @@ -16,10 +16,17 @@ export RULES_PYTHON_BOOTSTRAP_VERBOSE=1 # We're testing the invocation of `__main__.py`, so we have to # manually pass the zipapp to python. +echo "=====================================================================" echo "Running zipapp using an automatic temp directory..." +echo "=====================================================================" "$PYTHON" "$ZIPAPP" +echo +echo + +echo "=====================================================================" echo "Running zipapp with extract root set..." +echo "=====================================================================" export RULES_PYTHON_EXTRACT_ROOT="${TEST_TMPDIR:-/tmp}/extract_root_test" "$PYTHON" "$ZIPAPP" @@ -46,5 +53,7 @@ if [ ! -d "$RULES_PYTHON_EXTRACT_ROOT/$EXTRACT_DIR"/*/runfiles ]; then exit 1 fi +echo "=====================================================================" echo "Running zipapp with extract root set a second time..." +echo "=====================================================================" "$PYTHON" "$ZIPAPP" From 04794e8cb091a6c81345d743c2b67b35c763b153 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 5 Apr 2026 09:46:36 -0700 Subject: [PATCH 9/9] fix long path error --- python/private/zipapp/zip_main_template.py | 24 +++++++++++-------- ...m_python_zipapp_external_bootstrap_test.sh | 21 +++++++--------- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/python/private/zipapp/zip_main_template.py b/python/private/zipapp/zip_main_template.py index ca0ade34eb..97c37fee0b 100644 --- a/python/private/zipapp/zip_main_template.py +++ b/python/private/zipapp/zip_main_template.py @@ -27,7 +27,7 @@ import subprocess import tempfile import zipfile -from os.path import dirname, join +from os.path import dirname, join, basename # runfiles-root-relative path _STAGE2_BOOTSTRAP = "%stage2_bootstrap%" @@ -42,6 +42,7 @@ APP_HASH = "%APP_HASH%" EXTRACT_ROOT = os.environ.get("RULES_PYTHON_EXTRACT_ROOT") +IS_WINDOWS = os.name == "nt" def print_verbose(*args, mapping=None, values=None): @@ -68,10 +69,6 @@ def print_verbose(*args, mapping=None, values=None): print("bootstrap: stage 1:", *args, file=sys.stderr, flush=True) -# Return True if running on Windows -def is_windows(): - return os.name == "nt" - def get_windows_path_with_unc_prefix(path): """Adds UNC prefix after getting a normalized absolute Windows path. @@ -82,7 +79,7 @@ def get_windows_path_with_unc_prefix(path): # No need to add prefix for non-Windows platforms. # And \\?\ doesn't work in python 2 or on mingw - if not is_windows() or sys.version_info[0] < 3: + if not IS_WINDOWS or sys.version_info[0] < 3: return path # Starting in Windows 10, version 1607(OS build 14393), MAX_PATH limitations have been @@ -114,7 +111,7 @@ def has_windows_executable_extension(path): if ( _PYTHON_BINARY_VENV - and is_windows() + and IS_WINDOWS and not has_windows_executable_extension(_PYTHON_BINARY_VENV) ): _PYTHON_BINARY_VENV = _PYTHON_BINARY_VENV + ".exe" @@ -198,7 +195,14 @@ def extract_zip(zip_path, dest_dir): # Create the runfiles tree by extracting the zip file def create_runfiles_root(): if EXTRACT_ROOT: - extract_root = join(EXTRACT_ROOT, EXTRACT_DIR, APP_HASH) + # Shorten the path for Windows in case long path support is disabled + if IS_WINDOWS: + hash_dir = APP_HASH[0:32] + extract_dir = basename(EXTRACT_DIR) + extract_root = join(EXTRACT_ROOT, extract_dir, hash_dir) + else: + extract_root = join(EXTRACT_ROOT, EXTRACT_DIR, APP_HASH) + extract_root = get_windows_path_with_unc_prefix(extract_root) else: extract_root = tempfile.mkdtemp("", "Bazel.runfiles_") extract_zip(dirname(__file__), extract_root) @@ -246,9 +250,9 @@ def execute_file( subprocess_argv.append(f"-XRULES_PYTHON_ZIP_DIR={dirname(runfiles_root)}") subprocess_argv.append(main_filename) subprocess_argv += args - print_verbose("subprocess argv:", values=subprocess_argv) print_verbose("subprocess env:", mapping=env) print_verbose("subprocess cwd:", workspace) + print_verbose("subprocess argv:", values=subprocess_argv) ret_code = subprocess.call(subprocess_argv, env=env, cwd=workspace) sys.exit(ret_code) finally: @@ -278,7 +282,7 @@ def main(): # The main Python source file. main_rel_path = _STAGE2_BOOTSTRAP - if is_windows(): + if IS_WINDOWS: main_rel_path = main_rel_path.replace("/", os.sep) runfiles_root = create_runfiles_root() diff --git a/tests/py_zipapp/system_python_zipapp_external_bootstrap_test.sh b/tests/py_zipapp/system_python_zipapp_external_bootstrap_test.sh index cc5694285f..e7396007d9 100755 --- a/tests/py_zipapp/system_python_zipapp_external_bootstrap_test.sh +++ b/tests/py_zipapp/system_python_zipapp_external_bootstrap_test.sh @@ -36,20 +36,15 @@ if [[ ! -d "$RULES_PYTHON_EXTRACT_ROOT" ]]; then exit 1 fi -# The extract dir is _main/tests/py_zipapp/system_python_zipapp -# The new structure should be $RULES_PYTHON_EXTRACT_ROOT/_main/tests/py_zipapp/system_python_zipapp//runfiles -# We check that there is a subdirectory under the expected extract dir. -EXTRACT_DIR="_main/tests/py_zipapp/system_python_zipapp" -if [[ ! -d "$RULES_PYTHON_EXTRACT_ROOT/$EXTRACT_DIR" ]]; then - echo "Error: Extract directory $RULES_PYTHON_EXTRACT_ROOT/$EXTRACT_DIR was not created!" - exit 1 -fi - -# Check for the extra hash component. -# We use glob expansion to check for the expected depth. +# On windows, the path is shortened to just the basename to avoid long path errors. +# Other platforms use the full path. # Note: [ -d ... ] expands globs, while [[ -d ... ]] does not. -if [ ! -d "$RULES_PYTHON_EXTRACT_ROOT/$EXTRACT_DIR"/*/runfiles ]; then - echo "Error: Could not find 'runfiles' directory at expected depth $RULES_PYTHON_EXTRACT_ROOT/$EXTRACT_DIR/*/runfiles" +if [ -d "$RULES_PYTHON_EXTRACT_ROOT/_main/tests/py_zipapp/system_python_zipapp"/*/runfiles ]; then + echo "Found runfiles at $RULES_PYTHON_EXTRACT_ROOT/_main/tests/py_zipapp/system_python_zipapp/*/runfiles" +elif [ -d "$RULES_PYTHON_EXTRACT_ROOT/system_python_zipapp"/*/runfiles ]; then + echo "Found runfiles at $RULES_PYTHON_EXTRACT_ROOT/system_python_zipapp/*/runfiles" +else + echo "Error: Could not find 'runfiles' directory" exit 1 fi