From b63d4ad736428395dca3a8f2cdbb34ebe2836961 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Wed, 30 Mar 2022 14:09:52 -0700 Subject: [PATCH 1/9] Remove the nobuild/partialbuildmethod tests from python_bindings/ They no longer serve a purpose and are redundant to other tests. --- python_bindings/Makefile | 2 +- python_bindings/correctness/CMakeLists.txt | 2 - .../correctness/nobuildmethod_generator.cpp | 24 --------- .../partialbuildmethod_generator.cpp | 49 ------------------- python_bindings/correctness/pystub.py | 37 -------------- 5 files changed, 1 insertion(+), 113 deletions(-) delete mode 100644 python_bindings/correctness/nobuildmethod_generator.cpp delete mode 100644 python_bindings/correctness/partialbuildmethod_generator.cpp diff --git a/python_bindings/Makefile b/python_bindings/Makefile index 6c22e816304b..bc96a636646d 100644 --- a/python_bindings/Makefile +++ b/python_bindings/Makefile @@ -164,7 +164,7 @@ $(BIN)/ext/%.so: $(BIN)/%.py.o $(BIN)/%.a $(BIN)/runtime.a $(BIN)/ext/%.ldscript test_correctness_addconstant_test: $(BIN)/ext/addconstant.so test_correctness_bit_test: $(BIN)/ext/bit.so test_correctness_user_context_test: $(BIN)/ext/user_context.so -test_correctness_pystub: $(BIN)/generators/simplestub.so $(BIN)/generators/complexstub.so $(BIN)/generators/partialbuildmethod.so $(BIN)/generators/nobuildmethod.so +test_correctness_pystub: $(BIN)/generators/simplestub.so $(BIN)/generators/complexstub.so APPS = $(shell ls $(ROOT_DIR)/apps/*.py) CORRECTNESS = $(shell ls $(ROOT_DIR)/correctness/*.py) diff --git a/python_bindings/correctness/CMakeLists.txt b/python_bindings/correctness/CMakeLists.txt index 8c760705f3b0..e3085cca887e 100644 --- a/python_bindings/correctness/CMakeLists.txt +++ b/python_bindings/correctness/CMakeLists.txt @@ -1,7 +1,5 @@ set(GENERATORS complexstub_generator.cpp - nobuildmethod_generator.cpp - partialbuildmethod_generator.cpp simplestub_generator.cpp ) diff --git a/python_bindings/correctness/nobuildmethod_generator.cpp b/python_bindings/correctness/nobuildmethod_generator.cpp deleted file mode 100644 index 2d515e5a761b..000000000000 --- a/python_bindings/correctness/nobuildmethod_generator.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#include "Halide.h" - -namespace { - -// This Generator exists solely to compare the output with BuildMethod and PartialBuildMethod. -class NoBuildMethod : public Halide::Generator { -public: - GeneratorParam compiletime_factor{"compiletime_factor", 1, 0, 100}; - - Input> input{"input"}; - Input runtime_factor{"runtime_factor", 1.0}; - Output> output{"output"}; - - void generate() { - Var x, y; - - output(x, y) = - cast(input(x, y) * compiletime_factor * runtime_factor); - } -}; - -} // namespace - -HALIDE_REGISTER_GENERATOR(NoBuildMethod, nobuildmethod) diff --git a/python_bindings/correctness/partialbuildmethod_generator.cpp b/python_bindings/correctness/partialbuildmethod_generator.cpp deleted file mode 100644 index cc5f7cf2516e..000000000000 --- a/python_bindings/correctness/partialbuildmethod_generator.cpp +++ /dev/null @@ -1,49 +0,0 @@ -#include "Halide.h" - -namespace { - -#ifdef HALIDE_ALLOW_GENERATOR_BUILD_METHOD -// This Generator exists solely to test converted old-style -// generators -- which use Input<> rather than Param/ImageParam, but *don't* use -// Output<>/generate(). -// -// Do not convert it to new-style until/unless we decide to entirely remove -// support for those Generators. -class PartialBuildMethod : public Halide::Generator { -public: - GeneratorParam compiletime_factor{"compiletime_factor", 1, 0, 100}; - - Input> input{"input"}; - Input runtime_factor{"runtime_factor", 1.0}; - - Func build() { - Var x, y; - - Func g; - g(x, y) = - cast(input(x, y) * compiletime_factor * runtime_factor); - return g; - } -}; -#else -// Provide a placeholder here that uses generate(), just to allow this test to -// succeed even if build() is disabled. -class PartialBuildMethod : public Halide::Generator { -public: - GeneratorParam compiletime_factor{"compiletime_factor", 1, 0, 100}; - - Input> input{"input"}; - Input runtime_factor{"runtime_factor", 1.0}; - Output> output{"output"}; - - void generate() { - Var x, y; - - output(x, y) = cast(input(x, y) * compiletime_factor * runtime_factor); - } -}; -#endif - -} // namespace - -HALIDE_REGISTER_GENERATOR(PartialBuildMethod, partialbuildmethod) diff --git a/python_bindings/correctness/pystub.py b/python_bindings/correctness/pystub.py index ed284c9cf5b8..280de4a5359c 100644 --- a/python_bindings/correctness/pystub.py +++ b/python_bindings/correctness/pystub.py @@ -5,9 +5,6 @@ # test alternate-but-legal syntax from complexstub import generate as complexstub -import partialbuildmethod -import nobuildmethod - def _realize_and_check(f, offset = 0): b = hl.Buffer(hl.Float(32), [2, 2]) f.realize(b) @@ -260,41 +257,7 @@ def test_complexstub(): actual = b[x, y] assert expected == actual, "Expected %s Actual %s" % (expected, actual) -# disabled because HALIDE_ALLOW_GENERATOR_BUILD_METHOD is off by default -def test_partialbuildmethod(): - x, y, c = hl.Var(), hl.Var(), hl.Var() - target = hl.get_jit_target_from_environment() - - b_in = hl.Buffer(hl.Float(32), [2, 2]) - b_in.fill(123) - - b_out = hl.Buffer(hl.Int(32), [2, 2]) - - try: - f = partialbuildmethod.generate(target, b_in, 1.0) - except RuntimeError as e: - assert "Generators that use build() (instead of generate()+Output<>) are not supported in the Python bindings." in str(e) - else: - assert False, 'Did not see expected exception!' - -def test_nobuildmethod(): - x, y, c = hl.Var(), hl.Var(), hl.Var() - target = hl.get_jit_target_from_environment() - - b_in = hl.Buffer(hl.Float(32), [2, 2]) - b_in.fill(123) - - b_out = hl.Buffer(hl.Int(32), [2, 2]) - - f = nobuildmethod.generate(target, b_in, 1.0) - f.realize(b_out) - - assert b_out.all_equal(123) - if __name__ == "__main__": test_simplestub() test_looplevel() test_complexstub() - # disabled because HALIDE_ALLOW_GENERATOR_BUILD_METHOD is off by default - # test_partialbuildmethod() - test_nobuildmethod() From f9764fbad14fd26876ca766c270692d9138b2906 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Wed, 30 Mar 2022 15:41:52 -0700 Subject: [PATCH 2/9] WIP --- python_bindings/Makefile | 150 +++++++++++------- python_bindings/correctness/CMakeLists.txt | 25 +-- .../correctness/ext/CMakeLists.txt | 37 ----- .../correctness/generators/CMakeLists.txt | 132 +++++++++++++++ .../addconstant_generator.cpp | 0 .../{ => generators}/bit_generator.cpp | 0 .../complex_generator.cpp} | 4 +- .../{ext => generators}/ext.ldscript.apple.in | 0 .../{ext => generators}/ext.ldscript.linux.in | 0 .../simple_generator.cpp} | 4 +- .../user_context_generator.cpp | 0 python_bindings/correctness/pystub.py | 71 ++++----- .../stub/AddHalideGeneratorPython.cmake | 14 -- .../stub/AddPythonStubExtension.cmake | 23 +++ python_bindings/stub/CMakeLists.txt | 2 +- 15 files changed, 297 insertions(+), 165 deletions(-) delete mode 100644 python_bindings/correctness/ext/CMakeLists.txt create mode 100644 python_bindings/correctness/generators/CMakeLists.txt rename python_bindings/correctness/{ => generators}/addconstant_generator.cpp (100%) rename python_bindings/correctness/{ => generators}/bit_generator.cpp (100%) rename python_bindings/correctness/{complexstub_generator.cpp => generators/complex_generator.cpp} (97%) rename python_bindings/correctness/{ext => generators}/ext.ldscript.apple.in (100%) rename python_bindings/correctness/{ext => generators}/ext.ldscript.linux.in (100%) rename python_bindings/correctness/{simplestub_generator.cpp => generators/simple_generator.cpp} (87%) rename python_bindings/correctness/{ => generators}/user_context_generator.cpp (100%) delete mode 100644 python_bindings/stub/AddHalideGeneratorPython.cmake create mode 100644 python_bindings/stub/AddPythonStubExtension.cmake diff --git a/python_bindings/Makefile b/python_bindings/Makefile index bc96a636646d..6e263f2a4d20 100644 --- a/python_bindings/Makefile +++ b/python_bindings/Makefile @@ -68,103 +68,138 @@ $(BIN)/src/%.o: $(ROOT_DIR)/src/%.cpp @mkdir -p $(@D) @$(CXX) $(CCFLAGS) -c $< -o $@ - -$(BIN)/%_generator.o: $(ROOT_DIR)/correctness/%_generator.cpp $(HALIDE_DISTRIB_PATH)/include/Halide.h - @echo Building $@... - @mkdir -p $(@D) - @$(CXX) $(CCFLAGS) -c $< -o $@ - -$(BIN)/PyStubImpl.o: $(ROOT_DIR)/stub/PyStubImpl.cpp $(HALIDE_DISTRIB_PATH)/include/Halide.h +$(BIN)/%_generator.o: $(ROOT_DIR)/correctness/generators/%_generator.cpp $(HALIDE_DISTRIB_PATH)/include/Halide.h @echo Building $@... @mkdir -p $(@D) @$(CXX) $(CCFLAGS) -c $< -o $@ -# Produce a Python extension for a C++ generator by compiling PyStub.cpp -# (with HALIDE_PYSTUB_GENERATOR_NAME defined to the Generator's build name), -# and linking with the generator's .o file, PyStubImpl.o, plus the same libHalide -# being used by halide.so. +# There are two sorts of Python Extensions that we can produce for a Halide Generator +# written in C++: # -# You can optionally also define HALIDE_PYSTUB_MODULE_NAME if you want the Python -# module name to be something other than the Generator build name. -$(BIN)/%_PyStub.o: $(ROOT_DIR)/stub/PyStub.cpp - @echo Building $@... - @mkdir -p $(@D) - @$(CXX) $(CCFLAGS) -DHALIDE_PYSTUB_GENERATOR_NAME=$* -c $< -o $@ +# - One that is essentially the 'native code' output of a Generator, wrapped with enough CPython +# glue code to make it callable from Python. This is analogous to the usual Generator output +# when building a C++ codebase, and is the usual mode used for distribution of final product; +# these correspond to 'ahead-of-time' (AOT) code generation. The resulting code has no dependency +# on libHalide. We'll refer to this sort of extension as an "AOT extension". +# +# - One that essentially *the Generator itself*, wrapped in CPython glue code to make it callable +# from Python at Halide compilation time. This is analogous to the (rarely used) GeneratorStub +# code that can be used to compose multiple Generators together. The resulting extension *does* +# depend on libHalide, and can be used in either JIT or AOT mode for compilation. +# We'll refer to this sort of extension as a "Stub extension". +# +# For testing purposes here, we don't bother using distutils/setuptools to produce a properly-packaged +# Python extension; rather, we simply produce a .so file with the correct name exported, and ensure +# it's in the PYTHONPATH when testing. +# +# In our build files here, we build both kinds of extension for every Generator in the generators/ +# directory (even though not all are used). As a simplistic way to distinguish between the two +# sorts of extensions, we use the unadorned Generator name for AOT extensions, and the Generator name +# suffixed with "_stub" for Stub extensions. (TODO: this is unsatisfyingly hackish; better suggestions +# would be welcome.) -$(BIN)/generators/%.so: $(BIN)/%_PyStub.o $(BIN)/PyStubImpl.o $(BIN)/%_generator.o $(LIBHALIDE) +$(BIN)/PyStubImpl.o: $(ROOT_DIR)/stub/PyStubImpl.cpp $(HALIDE_DISTRIB_PATH)/include/Halide.h @echo Building $@... @mkdir -p $(@D) - @$(CXX) $^ $(LDFLAGS) -shared -o $@ + @$(CXX) $(CCFLAGS) -c $< -o $@ # Compile the generators: -$(BIN)/%.gen: $(HALIDE_DISTRIB_PATH)/tools/GenGen.cpp $(BIN)/%_generator.o $(LIBHALIDE) +$(BIN)/%.generator: $(HALIDE_DISTRIB_PATH)/tools/GenGen.cpp $(BIN)/%_generator.o $(LIBHALIDE) @echo Building $@... @mkdir -p $(@D) @$(CXX) $(CCFLAGS) $(LDFLAGS) $^ -o $@ # Special generator for generating a runtime: -$(BIN)/runtime.gen: $(HALIDE_DISTRIB_PATH)/tools/GenGen.cpp $(LIBHALIDE) +$(BIN)/runtime.generator: $(HALIDE_DISTRIB_PATH)/tools/GenGen.cpp $(LIBHALIDE) @echo Building $@... @mkdir -p $(@D) @$(CXX) $(CCFLAGS) $(LDFLAGS) $^ -o $@ # Generate a runtime: -$(BIN)/runtime.a: $(BIN)/runtime.gen +$(BIN)/runtime.a: $(BIN)/runtime.generator @echo Building $@... @mkdir -p $(@D) @$< -r runtime -o $(BIN) target=host -# Which target features to use for which test targets. -target_features_addconstant=-no_runtime -target_features_bit=-no_runtime -target_features_user_context=-user_context-no_runtime +# Construct linker script that will export *just* the PyInit entry +# point we want. (If we don't do this we can have interesting failures +# when loading multiple of these Python extensions in the same space.) +ifeq ($(UNAME), Darwin) +$(BIN)/%.ldscript: correctness/generators/ext.ldscript.apple.in + @echo Building $@... + @mkdir -p $(@D) + @cat $< | sed 's/$${GEN}/$*/' > $@ +PYEXT_LDSCRIPT_FLAG = -Wl,-exported_symbols_list %LDSCRIPT% +else +# Assume Desktop Linux +$(BIN)/%.ldscript: correctness/generators/ext.ldscript.linux.in + @echo Building $@... + @mkdir -p $(@D) + @cat $< | sed 's/$${GEN}/$*/' > $@ +PYEXT_LDSCRIPT_FLAG = -Wl,--version-script=%LDSCRIPT% +endif -# Make the generator generate a Python extension: -$(BIN)/%.py.cpp $(BIN)/%.a $(BIN)/%.h: $(BIN)/%.gen +# Some Generators require extra Halide Target Features to be set. +FEATURES_user_context=-user_context + +# Some Generators have undefined types, sizes, etc that are useful for Stubs extensions, +# but unacceptable for AOT Extensions; ensure that all of those are explicitly +# specified for AOT. (We currently don't use or test these in AOT form, so the settings +# are somewhat arbitrary.) +GENPARAMS_complex=\ + array_input.size=2 \ + array_input.type=uint8 \ + int_arg.size=2 \ + simple_input.type=uint8 \ + untyped_buffer_input.type=uint8 + +GENPARAMS_simple=\ + func_input.type=uint8 + +# Run the Generator to produce a static library of AOT code, +# plus the 'python_extension' code necessary to produce a useful +# AOT Extention for Python: +$(BIN)/%.py.cpp $(BIN)/%.a $(BIN)/%.h: $(BIN)/%.generator @echo Building $@... @LD_LIBRARY_PATH=$(HALIDE_DISTRIB_PATH)/bin $< \ -e static_library,c_header,python_extension \ - -g $(notdir $(basename $<)) -o $(BIN) \ - target=host$(target_features_$(notdir $(basename $<))) + -g $(notdir $(basename $<)) \ + -o $(BIN) \ + target=host-no_runtime$(FEATURES_$(notdir $(basename $<))) \ + $(GENPARAMS_$(notdir $(basename $<))) # Compile the generated Python extension(s): $(BIN)/%.py.o: $(BIN)/%.py.cpp @echo Building $@... @$(CXX) -c $(FPIC) $(CCFLAGS) $^ -o $@ -# Fake up a linker script that will export *just* the PyInit entry -# point we want. (If we don't do this we can have interesting failures -# when loading multiple of these Python extensions in the same space.) -ifeq ($(UNAME), Darwin) -$(BIN)/ext/%.ldscript: +# We take the native-code output of the Generator, add the Python-Extension +# code (to make it callable from Python), and put the resulting Python AOT Extension +# into the 'aot' folder. +$(BIN)/aot/%.so: $(BIN)/%.py.o $(BIN)/%.a $(BIN)/runtime.a $(BIN)/%.ldscript @echo Building $@... @mkdir -p $(@D) - @echo _PyInit_$* > $@ + @$(CXX) $(LDFLAGS) $(filter-out %.ldscript,$^) -shared $(subst %LDSCRIPT%,$(BIN)/$*.ldscript,$(PYEXT_LDSCRIPT_FLAG)) -o $@ -PYEXT_LDSCRIPT_FLAG = -Wl,-exported_symbols_list %LDSCRIPT% -else -# Assume Desktop Linux -$(BIN)/ext/%.ldscript: +# OK, now we want to produce a Stub Extension for the same Generator: +# Compiling PyStub.cpp, then linking with the generator's .o file, PyStubImpl.o, plus the same libHalide +# being used by halide.so. +# +# Note that we set HALIDE_PYSTUB_MODULE_NAME to $*_stub (e.g. foo_stub) but +# set HALIDE_PYSTUB_GENERATOR_NAME to the unadorned name of the Generator. +$(BIN)/%_PyStub.o: $(ROOT_DIR)/stub/PyStub.cpp @echo Building $@... @mkdir -p $(@D) - @echo "{" > $@ - @echo " global: PyInit_$*;" >> $@ - @echo " local: *;" >> $@ - @echo "};" >> $@ -PYEXT_LDSCRIPT_FLAG = -Wl,--version-script=%LDSCRIPT% -endif + @$(CXX) $(CCFLAGS) -DHALIDE_PYSTUB_MODULE_NAME=$*_stub -DHALIDE_PYSTUB_GENERATOR_NAME=$* -c $< -o $@ -# The Python extension of the generator is already in $(BIN), and is named -# the same, so put the Python extension of the function into ext/. -$(BIN)/ext/%.so: $(BIN)/%.py.o $(BIN)/%.a $(BIN)/runtime.a $(BIN)/ext/%.ldscript +$(BIN)/stub/%_stub.so: $(BIN)/%_PyStub.o $(BIN)/PyStubImpl.o $(BIN)/%_generator.o $(BIN)/%_stub.ldscript $(LIBHALIDE) @echo Building $@... @mkdir -p $(@D) - @$(CXX) $(LDFLAGS) $(filter-out $(BIN)/ext/$*.ldscript,$^) -shared $(subst %LDSCRIPT%,$(BIN)/ext/$*.ldscript,$(PYEXT_LDSCRIPT_FLAG)) -o $@ + @$(CXX) $(LDFLAGS) $(filter-out %.ldscript,$^) -shared $(subst %LDSCRIPT%,$(BIN)/$*_stub.ldscript,$(PYEXT_LDSCRIPT_FLAG)) -o $@ -test_correctness_addconstant_test: $(BIN)/ext/addconstant.so -test_correctness_bit_test: $(BIN)/ext/bit.so -test_correctness_user_context_test: $(BIN)/ext/user_context.so -test_correctness_pystub: $(BIN)/generators/simplestub.so $(BIN)/generators/complexstub.so +GENERATOR_SRCS=$(shell ls $(ROOT_DIR)/correctness/generators/*_generator.cpp) +GENERATOR_AOT_EXTENSIONS=$(GENERATOR_SRCS:$(ROOT_DIR)/correctness/generators/%_generator.cpp=$(BIN)/aot/%.so) +GENERATOR_STUB_EXTENSIONS=$(GENERATOR_SRCS:$(ROOT_DIR)/correctness/generators/%_generator.cpp=$(BIN)/stub/%_stub.so) APPS = $(shell ls $(ROOT_DIR)/apps/*.py) CORRECTNESS = $(shell ls $(ROOT_DIR)/correctness/*.py) @@ -182,10 +217,13 @@ test_apps_%: $(ROOT_DIR)/apps/%.py $(MODULE) .PHONY: test_correctness test_correctness: $(CORRECTNESS:$(ROOT_DIR)/correctness/%.py=test_correctness_%) -test_correctness_%: $(ROOT_DIR)/correctness/%.py $(MODULE) +# For simplicity of the build system, we just have every correctness test depend +# on every Generator and AOT extension. Normally this would not be a good idea, +# but it's fine for us here. +test_correctness_%: $(ROOT_DIR)/correctness/%.py $(MODULE) $(GENERATOR_AOT_EXTENSIONS) $(GENERATOR_STUB_EXTENSIONS) @echo Testing $*... @mkdir -p $(TEST_TMP) - @cd $(TEST_TMP); PYTHONPATH="$(BIN)/ext:$(BIN)/generators:$(BIN):$$PYTHONPATH" $(PYTHON) $< + @cd $(TEST_TMP); PYTHONPATH="$(BIN)/aot:$(BIN)/stub:$(BIN):$$PYTHONPATH" $(PYTHON) $< .PHONY: test_tutorial test_tutorial: $(TUTORIAL:$(ROOT_DIR)/tutorial/%.py=test_tutorial_%) diff --git a/python_bindings/correctness/CMakeLists.txt b/python_bindings/correctness/CMakeLists.txt index e3085cca887e..ec0784a14a67 100644 --- a/python_bindings/correctness/CMakeLists.txt +++ b/python_bindings/correctness/CMakeLists.txt @@ -1,15 +1,4 @@ -set(GENERATORS - complexstub_generator.cpp - simplestub_generator.cpp - ) - -foreach (GEN IN LISTS GENERATORS) - string(REPLACE "_generator.cpp" "" TARGET "${GEN}") - add_generator_python(${TARGET} ${GEN}) -endforeach () - -# Handle addconstant, bit, user_context -add_subdirectory(ext) +add_subdirectory(generators) add_library(the_sort_function MODULE the_sort_function.c) target_link_libraries(the_sort_function PRIVATE Halide::Runtime) @@ -38,12 +27,14 @@ set(TESTS var.py ) -# Use generator expressions to get the true output paths of these files -# CMAKE_CURRENT_BINARY_DIR is incorrect. +# Use generator expressions to get the true output paths of these files. make_shell_path(PYTHONPATH - "$" - "$" - "$") + "$" + "$" + "$" + ) + +message(STATUS "PYTHONPATH=${PYTHONPATH}") foreach (TEST IN LISTS TESTS) get_filename_component(TEST_NAME ${TEST} NAME_WE) diff --git a/python_bindings/correctness/ext/CMakeLists.txt b/python_bindings/correctness/ext/CMakeLists.txt deleted file mode 100644 index 40d41a23c5b5..000000000000 --- a/python_bindings/correctness/ext/CMakeLists.txt +++ /dev/null @@ -1,37 +0,0 @@ -include(TargetExportScript) - -set(FEATURES_user_context user_context) - -foreach (GEN IN ITEMS addconstant bit user_context) - # Create the Halide generator executable - add_executable(${GEN}.gen ../${GEN}_generator.cpp) - target_link_libraries(${GEN}.gen PRIVATE Halide::Generator) - - # Call it to generate the Python extension cpp file - add_halide_library(ext_${GEN} - FROM ${GEN}.gen - GENERATOR ${GEN} - FUNCTION_NAME ${GEN} - PYTHON_EXTENSION ${GEN}_py_cpp - FEATURES ${FEATURES_${GEN}} - TARGETS cmake) - - # Create the module from the generated library and .py.cpp - Python3_add_library(py_${GEN} MODULE ${${GEN}_py_cpp}) - target_link_libraries(py_${GEN} PRIVATE ext_${GEN}) - set_target_properties(py_${GEN} PROPERTIES OUTPUT_NAME ${GEN}) # Python3_add_library adds target info to name. - - # Fake up a linker script that will export *just* the PyInit entry - # point we want. (If we don't do this we can have interesting failures - # when loading multiple of these Python extensions in the same space.) - # - # TODO: How to do this for Windows as well? - configure_file(ext.ldscript.apple.in "${CMAKE_CURRENT_BINARY_DIR}/${GEN}.ldscript.apple") - configure_file(ext.ldscript.linux.in "${CMAKE_CURRENT_BINARY_DIR}/${GEN}.ldscript") - target_export_script( - py_${GEN} - APPLE_LD "${CMAKE_CURRENT_BINARY_DIR}/${GEN}.ldscript.apple" - GNU_LD "${CMAKE_CURRENT_BINARY_DIR}/${GEN}.ldscript" - ) - -endforeach () diff --git a/python_bindings/correctness/generators/CMakeLists.txt b/python_bindings/correctness/generators/CMakeLists.txt new file mode 100644 index 000000000000..ce582477c001 --- /dev/null +++ b/python_bindings/correctness/generators/CMakeLists.txt @@ -0,0 +1,132 @@ +include(TargetExportScript) + +# There are two sorts of Python Extensions that we can produce for a Halide Generator +# written in C++: +# +# - One that is essentially the 'native code' output of a Generator, wrapped with enough CPython +# glue code to make it callable from Python. This is analogous to the usual Generator output +# when building a C++ codebase, and is the usual mode used for distribution of final product; +# these correspond to 'ahead-of-time' (AOT) code generation. The resulting code has no dependency +# on libHalide. We'll refer to this sort of extension as an "AOT extension". +# +# - One that essentially *the Generator itself*, wrapped in CPython glue code to make it callable +# from Python at Halide compilation time. This is analogous to the (rarely used) GeneratorStub +# code that can be used to compose multiple Generators together. The resulting extension *does* +# depend on libHalide, and can be used in either JIT or AOT mode for compilation. +# We'll refer to this sort of extension as a "Stub extension". +# +# For testing purposes here, we don't bother using distutils/setuptools to produce a properly-packaged +# Python extension; rather, we simply produce a .so file with the correct name exported, and ensure +# it's in the PYTHONPATH when testing. +# +# In our build files here, we build both kinds of extension for every Generator in the generators/ +# directory (even though not all are used). As a simplistic way to distinguish between the two +# sorts of extensions, we use the unadorned Generator name for AOT extensions, and the Generator name +# suffixed with "_stub" for Stub extensions. (TODO: this is unsatisfyingly hackish; better suggestions +# would be welcome.) + +set(GENERATORS + addconstant + bit + complex + simple + user_context + ) + +# Some Generators require extra Halide Target Features to be set. +set(FEATURES_user_context user_context) + +# Some Generators have undefined types, sizes, etc that are useful for Stubs extensions, +# but unacceptable for AOT Extensions; ensure that all of those are explicitly +# specified for AOT. (We currently don't use or test these in AOT form, so the settings +# are somewhat arbitrary.) +set(GENPARAMS_complex + array_input.size=2 + array_input.type=uint8 + int_arg.size=2 + simple_input.type=uint8 + untyped_buffer_input.type=uint8) + +set(GENPARAMS_simple + func_input.type=uint8) + +# function(add_python_aot_extension TARGET) +# set(options) +# set(oneValueArgs GENERATOR MODULE) +# set(multiValueArgs) +# cmake_parse_arguments(ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + +# if (NOT ARG_GENERATOR) +# set(ARG_GENERATOR "${TARGET}") +# endif () + +# if (NOT ARG_MODULE) +# set(ARG_MODULE "${TARGET}_stub") +# endif () + +# Python3_add_library(${TARGET} MODULE ${HALIDE_PYSTUB_CPP_PATH}) +# target_compile_definitions(${TARGET} PRIVATE +# "HALIDE_PYSTUB_GENERATOR_NAME=${ARG_GENERATOR}" +# "HALIDE_PYSTUB_MODULE_NAME=${ARG_MODULE}") +# target_link_libraries(${TARGET} PRIVATE Halide::PyStubs) +# set_target_properties(${TARGET} PROPERTIES OUTPUT_NAME ${ARG_MODULE}) +# endfunction() + +foreach (GEN IN LISTS GENERATORS) + # Create the Halide generator executable. (Prepend "Py_" to avoid name + # conflicts with stuff elsewhere in Halide CMake files.) + add_executable(Py_${GEN}.generator ${GEN}_generator.cpp) + target_link_libraries(Py_${GEN}.generator PRIVATE Halide::Generator) + + # Run the Generator to produce a static library of AOT code, + # plus the 'python_extension' code necessary to produce a useful + # AOT Extention for Python: + add_halide_library(aot_${GEN} + FROM Py_${GEN}.generator + GENERATOR ${GEN} + FUNCTION_NAME ${GEN} + PYTHON_EXTENSION ${GEN}.py.cpp + FEATURES ${FEATURES_${GEN}} + PARAMS ${GENPARAMS_${GEN}} + TARGETS cmake) + + # Take the native-code output of the Generator, add the Python-Extension + # code (to make it callable from Python), and build it into the AOT Extension we need. + Python3_add_library(py_aot_${GEN} MODULE ${${GEN}.py.cpp}) + target_link_libraries(py_aot_${GEN} PRIVATE aot_${GEN}) + set_target_properties(py_aot_${GEN} PROPERTIES OUTPUT_NAME ${GEN}) # Python3_add_library adds target info to name. + + # Construct linker script that will export *just* the PyInit entry + # point we want. (If we don't do this we can have interesting failures + # when loading multiple of these Python extensions in the same space.) + # + # TODO: How to do this for Windows as well? + configure_file(ext.ldscript.apple.in "${CMAKE_CURRENT_BINARY_DIR}/${GEN}.ldscript.apple") + configure_file(ext.ldscript.linux.in "${CMAKE_CURRENT_BINARY_DIR}/${GEN}.ldscript") + target_export_script( + py_aot_${GEN} + APPLE_LD "${CMAKE_CURRENT_BINARY_DIR}/${GEN}.ldscript.apple" + GNU_LD "${CMAKE_CURRENT_BINARY_DIR}/${GEN}.ldscript" + ) + + # OK, now we want to produce a Stub Extension for the same Generator: + # Compiling PyStub.cpp, then linking with the generator's .o file, PyStubImpl.o, plus the same libHalide + # being used by halide.so. + # + # Note that we set HALIDE_PYSTUB_MODULE_NAME to $*_stub (e.g. foo_stub) but + # set HALIDE_PYSTUB_GENERATOR_NAME to the unadorned name of the Generator. + add_python_stub_extension(py_stub_${GEN} + GENERATOR ${GEN} + MODULE ${GEN}_stub) + + # Same trick with Linker scripts here. + configure_file(ext.ldscript.apple.in "${CMAKE_CURRENT_BINARY_DIR}/${GEN}_stub.ldscript.apple") + configure_file(ext.ldscript.linux.in "${CMAKE_CURRENT_BINARY_DIR}/${GEN}_stub.ldscript") + target_export_script( + py_stub_${GEN} + APPLE_LD "${CMAKE_CURRENT_BINARY_DIR}/${GEN}.ldscript.apple" + GNU_LD "${CMAKE_CURRENT_BINARY_DIR}/${GEN}.ldscript" + ) + + +endforeach () diff --git a/python_bindings/correctness/addconstant_generator.cpp b/python_bindings/correctness/generators/addconstant_generator.cpp similarity index 100% rename from python_bindings/correctness/addconstant_generator.cpp rename to python_bindings/correctness/generators/addconstant_generator.cpp diff --git a/python_bindings/correctness/bit_generator.cpp b/python_bindings/correctness/generators/bit_generator.cpp similarity index 100% rename from python_bindings/correctness/bit_generator.cpp rename to python_bindings/correctness/generators/bit_generator.cpp diff --git a/python_bindings/correctness/complexstub_generator.cpp b/python_bindings/correctness/generators/complex_generator.cpp similarity index 97% rename from python_bindings/correctness/complexstub_generator.cpp rename to python_bindings/correctness/generators/complex_generator.cpp index 1b6e3274a806..db27c2e96d19 100644 --- a/python_bindings/correctness/complexstub_generator.cpp +++ b/python_bindings/correctness/generators/complex_generator.cpp @@ -15,7 +15,7 @@ Halide::Buffer make_image(int extra) { return im; } -class ComplexStub : public Halide::Generator { +class Complex : public Halide::Generator { public: GeneratorParam untyped_buffer_output_type{"untyped_buffer_output_type", Float(32)}; GeneratorParam vectorize{"vectorize", true}; @@ -90,4 +90,4 @@ class ComplexStub : public Halide::Generator { } // namespace -HALIDE_REGISTER_GENERATOR(ComplexStub, complexstub) +HALIDE_REGISTER_GENERATOR(Complex, complex) diff --git a/python_bindings/correctness/ext/ext.ldscript.apple.in b/python_bindings/correctness/generators/ext.ldscript.apple.in similarity index 100% rename from python_bindings/correctness/ext/ext.ldscript.apple.in rename to python_bindings/correctness/generators/ext.ldscript.apple.in diff --git a/python_bindings/correctness/ext/ext.ldscript.linux.in b/python_bindings/correctness/generators/ext.ldscript.linux.in similarity index 100% rename from python_bindings/correctness/ext/ext.ldscript.linux.in rename to python_bindings/correctness/generators/ext.ldscript.linux.in diff --git a/python_bindings/correctness/simplestub_generator.cpp b/python_bindings/correctness/generators/simple_generator.cpp similarity index 87% rename from python_bindings/correctness/simplestub_generator.cpp rename to python_bindings/correctness/generators/simple_generator.cpp index c7f4e56b7de7..759d6184b627 100644 --- a/python_bindings/correctness/simplestub_generator.cpp +++ b/python_bindings/correctness/generators/simple_generator.cpp @@ -2,7 +2,7 @@ namespace { -class SimpleStub : public Halide::Generator { +class Simple : public Halide::Generator { public: GeneratorParam offset{"offset", 0}; GeneratorParam compute_level{"compute_level", LoopLevel::root()}; @@ -27,4 +27,4 @@ class SimpleStub : public Halide::Generator { } // namespace -HALIDE_REGISTER_GENERATOR(SimpleStub, simplestub) +HALIDE_REGISTER_GENERATOR(Simple, simple) diff --git a/python_bindings/correctness/user_context_generator.cpp b/python_bindings/correctness/generators/user_context_generator.cpp similarity index 100% rename from python_bindings/correctness/user_context_generator.cpp rename to python_bindings/correctness/generators/user_context_generator.cpp diff --git a/python_bindings/correctness/pystub.py b/python_bindings/correctness/pystub.py index 280de4a5359c..b1709ee75c60 100644 --- a/python_bindings/correctness/pystub.py +++ b/python_bindings/correctness/pystub.py @@ -1,9 +1,8 @@ import halide as hl -import simplestub -# test alternate-but-legal syntax -from complexstub import generate as complexstub +import simple_stub +import complex_stub def _realize_and_check(f, offset = 0): b = hl.Buffer(hl.Float(32), [2, 2]) @@ -15,7 +14,7 @@ def _realize_and_check(f, offset = 0): assert b[1, 1] == 5.5 + offset + 123 -def test_simplestub(): +def test_simple(gen): x, y = hl.Var(), hl.Var() target = hl.get_jit_target_from_environment() @@ -26,40 +25,40 @@ def test_simplestub(): f_in[x, y] = x + y # ----------- Inputs by-position - f = simplestub.generate(target, b_in, f_in, 3.5) + f = gen(target, b_in, f_in, 3.5) _realize_and_check(f) # ----------- Inputs by-name - f = simplestub.generate(target, buffer_input=b_in, func_input=f_in, float_arg=3.5) + f = gen(target, buffer_input=b_in, func_input=f_in, float_arg=3.5) _realize_and_check(f) - f = simplestub.generate(target, float_arg=3.5, buffer_input=b_in, func_input=f_in) + f = gen(target, float_arg=3.5, buffer_input=b_in, func_input=f_in) _realize_and_check(f) # ----------- Above set again, w/ GeneratorParam mixed in k = 42 # (positional) - f = simplestub.generate(target, b_in, f_in, 3.5, offset=k) + f = gen(target, b_in, f_in, 3.5, offset=k) _realize_and_check(f, k) # (keyword) - f = simplestub.generate(target, offset=k, buffer_input=b_in, func_input=f_in, float_arg=3.5) + f = gen(target, offset=k, buffer_input=b_in, func_input=f_in, float_arg=3.5) _realize_and_check(f, k) - f = simplestub.generate(target, buffer_input=b_in, offset=k, func_input=f_in, float_arg=3.5) + f = gen(target, buffer_input=b_in, offset=k, func_input=f_in, float_arg=3.5) _realize_and_check(f, k) - f = simplestub.generate(target, buffer_input=b_in, func_input=f_in, offset=k, float_arg=3.5) + f = gen(target, buffer_input=b_in, func_input=f_in, offset=k, float_arg=3.5) _realize_and_check(f, k) - f = simplestub.generate(target, buffer_input=b_in, float_arg=3.5, func_input=f_in, offset=k) + f = gen(target, buffer_input=b_in, float_arg=3.5, func_input=f_in, offset=k) _realize_and_check(f, k) # ----------- Test various failure modes try: # Inputs w/ mixed by-position and by-name - f = simplestub.generate(target, b_in, f_in, float_arg=3.5) + f = gen(target, b_in, f_in, float_arg=3.5) except RuntimeError as e: assert 'Cannot use both positional and keyword arguments for inputs.' in str(e) else: @@ -67,7 +66,7 @@ def test_simplestub(): try: # too many positional args - f = simplestub.generate(target, b_in, f_in, 3.5, 4) + f = gen(target, b_in, f_in, 3.5, 4) except RuntimeError as e: assert 'Expected exactly 3 positional args for inputs, but saw 4.' in str(e) else: @@ -75,7 +74,7 @@ def test_simplestub(): try: # too few positional args - f = simplestub.generate(target, b_in, f_in) + f = gen(target, b_in, f_in) except RuntimeError as e: assert 'Expected exactly 3 positional args for inputs, but saw 2.' in str(e) else: @@ -83,7 +82,7 @@ def test_simplestub(): try: # Inputs that can't be converted to what the receiver needs (positional) - f = simplestub.generate(target, hl.f32(3.141592), "happy", k) + f = gen(target, hl.f32(3.141592), "happy", k) except RuntimeError as e: assert 'Unable to cast Python instance' in str(e) else: @@ -91,7 +90,7 @@ def test_simplestub(): try: # Inputs that can't be converted to what the receiver needs (named) - f = simplestub.generate(target, b_in, f_in, float_arg="bogus") + f = gen(target, b_in, f_in, float_arg="bogus") except RuntimeError as e: assert 'Unable to cast Python instance' in str(e) else: @@ -99,7 +98,7 @@ def test_simplestub(): try: # Input specified by both pos and kwarg - f = simplestub.generate(target, b_in, f_in, 3.5, float_arg=4.5) + f = gen(target, b_in, f_in, 3.5, float_arg=4.5) except RuntimeError as e: assert "Cannot use both positional and keyword arguments for inputs." in str(e) else: @@ -107,7 +106,7 @@ def test_simplestub(): try: # Bad input name - f = simplestub.generate(target, buffer_input=b_in, float_arg=3.5, offset=k, funk_input=f_in) + f = gen(target, buffer_input=b_in, float_arg=3.5, offset=k, funk_input=f_in) except RuntimeError as e: assert "Expected exactly 3 keyword args for inputs, but saw 2." in str(e) else: @@ -115,13 +114,13 @@ def test_simplestub(): try: # Bad gp name - f = simplestub.generate(target, buffer_input=b_in, float_arg=3.5, offset=k, func_input=f_in, nonexistent_generator_param="wat") + f = gen(target, buffer_input=b_in, float_arg=3.5, offset=k, func_input=f_in, nonexistent_generator_param="wat") except RuntimeError as e: assert "Generator simplestub has no GeneratorParam named: nonexistent_generator_param" in str(e) else: assert False, 'Did not see expected exception!' -def test_looplevel(): +def test_looplevel(gen): x, y = hl.Var('x'), hl.Var('y') target = hl.get_jit_target_from_environment() @@ -132,7 +131,7 @@ def test_looplevel(): func_input[x, y] = x + y simple_compute_at = hl.LoopLevel() - simple = simplestub.generate(target, buffer_input, func_input, 3.5, + simple = gen(target, buffer_input, func_input, 3.5, compute_level=simple_compute_at) computed_output = hl.Func('computed_output') @@ -151,7 +150,7 @@ def _make_constant_image(): constant_image[x, y, c] = x + y + c return constant_image -def test_complexstub(): +def test_complexs(gen): constant_image = _make_constant_image() input = hl.ImageParam(hl.UInt(8), 3, 'input') input.set(constant_image) @@ -165,16 +164,16 @@ def test_complexstub(): func_input = hl.Func("func_input") func_input[x, y, c] = hl.u16(x + y + c) - r = complexstub(target, - typed_buffer_input=constant_image, - untyped_buffer_input=constant_image, - simple_input=input, - array_input=[ input, input ], - float_arg=float_arg, - int_arg=[ int_arg, int_arg ], - untyped_buffer_output_type="uint8", - extra_func_input=func_input, - vectorize=True) + r = gen(target, + typed_buffer_input=constant_image, + untyped_buffer_input=constant_image, + simple_input=input, + array_input=[ input, input ], + float_arg=float_arg, + int_arg=[ int_arg, int_arg ], + untyped_buffer_output_type="uint8", + extra_func_input=func_input, + vectorize=True) # return value is a tuple; unpack separately to avoid # making the callsite above unreadable @@ -258,6 +257,6 @@ def test_complexstub(): assert expected == actual, "Expected %s Actual %s" % (expected, actual) if __name__ == "__main__": - test_simplestub() - test_looplevel() - test_complexstub() + test_simple(simple_stub.generate) + test_looplevel(simple_stub.generate) + test_complex(complex_stub.generate) diff --git a/python_bindings/stub/AddHalideGeneratorPython.cmake b/python_bindings/stub/AddHalideGeneratorPython.cmake deleted file mode 100644 index 61322dedaf32..000000000000 --- a/python_bindings/stub/AddHalideGeneratorPython.cmake +++ /dev/null @@ -1,14 +0,0 @@ -set(HALIDE_PYSTUB_CPP_PATH ${CMAKE_CURRENT_LIST_DIR}/PyStub.cpp) - -function(add_generator_python TARGET) - set(options) - set(oneValueArgs) - set(multiValueArgs) - cmake_parse_arguments(args "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - - Python3_add_library(${TARGET} MODULE ${HALIDE_PYSTUB_CPP_PATH} ${args_UNPARSED_ARGUMENTS}) - target_compile_definitions(${TARGET} PRIVATE - "HALIDE_PYSTUB_GENERATOR_NAME=${TARGET}" - "HALIDE_PYSTUB_MODULE_NAME=${TARGET}") - target_link_libraries(${TARGET} PRIVATE Halide::PyStubs) -endfunction() diff --git a/python_bindings/stub/AddPythonStubExtension.cmake b/python_bindings/stub/AddPythonStubExtension.cmake new file mode 100644 index 000000000000..4d77b2d05085 --- /dev/null +++ b/python_bindings/stub/AddPythonStubExtension.cmake @@ -0,0 +1,23 @@ +set(HALIDE_PYSTUB_CPP_PATH ${CMAKE_CURRENT_LIST_DIR}/PyStub.cpp) + +function(add_python_stub_extension TARGET) + set(options) + set(oneValueArgs GENERATOR MODULE) + set(multiValueArgs) + cmake_parse_arguments(ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + if (NOT ARG_GENERATOR) + set(ARG_GENERATOR "${TARGET}") + endif () + + if (NOT ARG_MODULE) + set(ARG_MODULE "${TARGET}_stub") + endif () + + Python3_add_library(${TARGET} MODULE ${HALIDE_PYSTUB_CPP_PATH}) + target_compile_definitions(${TARGET} PRIVATE + "HALIDE_PYSTUB_GENERATOR_NAME=${ARG_GENERATOR}" + "HALIDE_PYSTUB_MODULE_NAME=${ARG_MODULE}") + target_link_libraries(${TARGET} PRIVATE Halide::PyStubs Halide::Halide) + set_target_properties(${TARGET} PROPERTIES OUTPUT_NAME ${ARG_MODULE}) +endfunction() diff --git a/python_bindings/stub/CMakeLists.txt b/python_bindings/stub/CMakeLists.txt index fedfda44e3df..3682eff6ee4a 100644 --- a/python_bindings/stub/CMakeLists.txt +++ b/python_bindings/stub/CMakeLists.txt @@ -10,4 +10,4 @@ if (NOT TARGET Halide::PyStubs) POSITION_INDEPENDENT_CODE ON) endif () -include(${CMAKE_CURRENT_LIST_DIR}/AddHalideGeneratorPython.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/AddPythonStubExtension.cmake) From fc18afafd14a03e0054b1bb8c62fe84df3b4a53a Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Wed, 30 Mar 2022 15:49:23 -0700 Subject: [PATCH 3/9] Update pystub.py --- python_bindings/correctness/pystub.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python_bindings/correctness/pystub.py b/python_bindings/correctness/pystub.py index b1709ee75c60..17e86139e56c 100644 --- a/python_bindings/correctness/pystub.py +++ b/python_bindings/correctness/pystub.py @@ -116,7 +116,7 @@ def test_simple(gen): # Bad gp name f = gen(target, buffer_input=b_in, float_arg=3.5, offset=k, func_input=f_in, nonexistent_generator_param="wat") except RuntimeError as e: - assert "Generator simplestub has no GeneratorParam named: nonexistent_generator_param" in str(e) + assert "has no GeneratorParam named: nonexistent_generator_param" in str(e) else: assert False, 'Did not see expected exception!' @@ -150,7 +150,7 @@ def _make_constant_image(): constant_image[x, y, c] = x + y + c return constant_image -def test_complexs(gen): +def test_complex(gen): constant_image = _make_constant_image() input = hl.ImageParam(hl.UInt(8), 3, 'input') input.set(constant_image) From 732285e984b3362cdecb86fbdf50a0b0c3147fbf Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Wed, 30 Mar 2022 15:56:17 -0700 Subject: [PATCH 4/9] wip --- python_bindings/correctness/CMakeLists.txt | 2 -- python_bindings/correctness/generators/CMakeLists.txt | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/python_bindings/correctness/CMakeLists.txt b/python_bindings/correctness/CMakeLists.txt index ec0784a14a67..84a729ec061f 100644 --- a/python_bindings/correctness/CMakeLists.txt +++ b/python_bindings/correctness/CMakeLists.txt @@ -34,8 +34,6 @@ make_shell_path(PYTHONPATH "$" ) -message(STATUS "PYTHONPATH=${PYTHONPATH}") - foreach (TEST IN LISTS TESTS) get_filename_component(TEST_NAME ${TEST} NAME_WE) add_test(NAME python_correctness_${TEST_NAME} diff --git a/python_bindings/correctness/generators/CMakeLists.txt b/python_bindings/correctness/generators/CMakeLists.txt index ce582477c001..beb82a414e0d 100644 --- a/python_bindings/correctness/generators/CMakeLists.txt +++ b/python_bindings/correctness/generators/CMakeLists.txt @@ -110,8 +110,8 @@ foreach (GEN IN LISTS GENERATORS) ) # OK, now we want to produce a Stub Extension for the same Generator: - # Compiling PyStub.cpp, then linking with the generator's .o file, PyStubImpl.o, plus the same libHalide - # being used by halide.so. + # Compiling PyStub.cpp, then linking with the generator's .o file, PyStubImpl.o, + # plus the same libHalide being used by halide.so. # # Note that we set HALIDE_PYSTUB_MODULE_NAME to $*_stub (e.g. foo_stub) but # set HALIDE_PYSTUB_GENERATOR_NAME to the unadorned name of the Generator. From 4db2540f8c82ca8bf5c1b2fb0ffcda1980ecb78a Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Wed, 30 Mar 2022 17:06:35 -0700 Subject: [PATCH 5/9] wip --- python_bindings/correctness/generators/CMakeLists.txt | 6 ++++++ python_bindings/stub/AddPythonStubExtension.cmake | 10 +++++++--- python_bindings/stub/CMakeLists.txt | 1 + 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/python_bindings/correctness/generators/CMakeLists.txt b/python_bindings/correctness/generators/CMakeLists.txt index beb82a414e0d..e0b0a8e8456d 100644 --- a/python_bindings/correctness/generators/CMakeLists.txt +++ b/python_bindings/correctness/generators/CMakeLists.txt @@ -78,6 +78,11 @@ foreach (GEN IN LISTS GENERATORS) add_executable(Py_${GEN}.generator ${GEN}_generator.cpp) target_link_libraries(Py_${GEN}.generator PRIVATE Halide::Generator) + # TODO: this should work (and would be preferred to the code above) + # but CMake fails with "targets not yet defined"; investigate. + # add_halide_generator(Py_${GEN}.generator + # SOURCES ${GEN}_generator.cpp) + # Run the Generator to produce a static library of AOT code, # plus the 'python_extension' code necessary to produce a useful # AOT Extention for Python: @@ -116,6 +121,7 @@ foreach (GEN IN LISTS GENERATORS) # Note that we set HALIDE_PYSTUB_MODULE_NAME to $*_stub (e.g. foo_stub) but # set HALIDE_PYSTUB_GENERATOR_NAME to the unadorned name of the Generator. add_python_stub_extension(py_stub_${GEN} + SOURCES ${GEN}_generator.cpp GENERATOR ${GEN} MODULE ${GEN}_stub) diff --git a/python_bindings/stub/AddPythonStubExtension.cmake b/python_bindings/stub/AddPythonStubExtension.cmake index 4d77b2d05085..96da764fff05 100644 --- a/python_bindings/stub/AddPythonStubExtension.cmake +++ b/python_bindings/stub/AddPythonStubExtension.cmake @@ -3,7 +3,7 @@ set(HALIDE_PYSTUB_CPP_PATH ${CMAKE_CURRENT_LIST_DIR}/PyStub.cpp) function(add_python_stub_extension TARGET) set(options) set(oneValueArgs GENERATOR MODULE) - set(multiValueArgs) + set(multiValueArgs SOURCES LINK_LIBRARIES) cmake_parse_arguments(ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) if (NOT ARG_GENERATOR) @@ -14,10 +14,14 @@ function(add_python_stub_extension TARGET) set(ARG_MODULE "${TARGET}_stub") endif () - Python3_add_library(${TARGET} MODULE ${HALIDE_PYSTUB_CPP_PATH}) + Python3_add_library(${TARGET} MODULE ${HALIDE_PYSTUB_CPP_PATH} ${ARG_SOURCES}) + set_target_properties(${TARGET} PROPERTIES + CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN ON + POSITION_INDEPENDENT_CODE ON) target_compile_definitions(${TARGET} PRIVATE "HALIDE_PYSTUB_GENERATOR_NAME=${ARG_GENERATOR}" "HALIDE_PYSTUB_MODULE_NAME=${ARG_MODULE}") - target_link_libraries(${TARGET} PRIVATE Halide::PyStubs Halide::Halide) + target_link_libraries(${TARGET} PRIVATE Halide::PyStubs ${ARG_LINK_LIBRARIES}) set_target_properties(${TARGET} PROPERTIES OUTPUT_NAME ${ARG_MODULE}) endfunction() diff --git a/python_bindings/stub/CMakeLists.txt b/python_bindings/stub/CMakeLists.txt index 3682eff6ee4a..90c1f487aca6 100644 --- a/python_bindings/stub/CMakeLists.txt +++ b/python_bindings/stub/CMakeLists.txt @@ -6,6 +6,7 @@ if (NOT TARGET Halide::PyStubs) target_link_libraries(Halide_PyStubs PUBLIC Halide::Halide) set_target_properties(Halide_PyStubs PROPERTIES EXPORT_NAME PyStubs + CXX_VISIBILITY_PRESET hidden VISIBILITY_INLINES_HIDDEN TRUE POSITION_INDEPENDENT_CODE ON) endif () From 2b8ec8397b6ad8f0da2448959be8e4d09047c9c2 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Wed, 30 Mar 2022 18:10:58 -0700 Subject: [PATCH 6/9] wip --- cmake/PythonExtensionHelpers.cmake | 115 ++++++++++++++++++ cmake/TargetExportScript.cmake | 5 +- python_bindings/CMakeLists.txt | 2 +- python_bindings/Makefile | 8 +- .../correctness/generators/CMakeLists.txt | 111 +---------------- .../generators/ext.ldscript.apple.in | 1 - .../generators/ext.ldscript.linux.in | 4 - .../stub/AddPythonStubExtension.cmake | 27 ---- python_bindings/stub/CMakeLists.txt | 2 - python_bindings/stub/ext.ldscript.apple.in | 1 + python_bindings/stub/ext.ldscript.linux.in | 4 + 11 files changed, 135 insertions(+), 145 deletions(-) create mode 100644 cmake/PythonExtensionHelpers.cmake delete mode 100644 python_bindings/correctness/generators/ext.ldscript.apple.in delete mode 100644 python_bindings/correctness/generators/ext.ldscript.linux.in delete mode 100644 python_bindings/stub/AddPythonStubExtension.cmake create mode 100644 python_bindings/stub/ext.ldscript.apple.in create mode 100644 python_bindings/stub/ext.ldscript.linux.in diff --git a/cmake/PythonExtensionHelpers.cmake b/cmake/PythonExtensionHelpers.cmake new file mode 100644 index 000000000000..71f50c2ea481 --- /dev/null +++ b/cmake/PythonExtensionHelpers.cmake @@ -0,0 +1,115 @@ +include(HalideGeneratorHelpers) +include(TargetExportScript) + +set(_STUB_DIR "${Halide_SOURCE_DIR}/python_bindings/stub") + +# There are two sorts of Python Extensions that we can produce for a Halide Generator +# written in C++: +# +# - One that is essentially the 'native code' output of a Generator, wrapped with enough CPython +# glue code to make it callable from Python. This is analogous to the usual Generator output +# when building a C++ codebase, and is the usual mode used for distribution of final product; +# these correspond to 'ahead-of-time' (AOT) code generation. The resulting code has no dependency +# on libHalide. We'll refer to this sort of extension as an "AOT extension". +# +# - One that essentially *the Generator itself*, wrapped in CPython glue code to make it callable +# from Python at Halide compilation time. This is analogous to the (rarely used) GeneratorStub +# code that can be used to compose multiple Generators together. The resulting extension *does* +# depend on libHalide, and can be used in either JIT or AOT mode for compilation. +# We'll refer to this sort of extension as a "Stub extension". +# +# For testing purposes here, we don't bother using distutils/setuptools to produce a properly-packaged +# Python extension; rather, we simply produce a .so file with the correct name exported, and ensure +# it's in the PYTHONPATH when testing. +# +# In our build files here, we build both kinds of extension for every Generator in the generators/ +# directory (even though not all are used). As a simplistic way to distinguish between the two +# sorts of extensions, we use the unadorned Generator name for AOT extensions, and the Generator name +# suffixed with "_stub" for Stub extensions. (TODO: this is unsatisfyingly hackish; better suggestions +# would be welcome.) + +function(target_export_single_symbol TARGET SYMBOL) + configure_file("${_STUB_DIR}/ext.ldscript.apple.in" "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}.ldscript.apple") + configure_file("${_STUB_DIR}/ext.ldscript.linux.in" "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}.ldscript") + target_export_script( + ${TARGET} + APPLE_LD "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}.ldscript.apple" + GNU_LD "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}.ldscript" + ) +endfunction() + +function(add_python_aot_extension TARGET) + set(options) + set(oneValueArgs GENERATOR FUNCTION_NAME) + set(multiValueArgs SOURCES LINK_LIBRARIES FEATURES PARAMS) + cmake_parse_arguments(ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + if (NOT ARG_GENERATOR) + set(ARG_GENERATOR "${TARGET}") + endif () + + if (NOT ARG_FUNCTION_NAME) + set(ARG_FUNCTION_NAME "${ARG_GENERATOR}") + endif () + + # Create the Halide generator executable. + add_executable(${TARGET}.generator ${ARG_SOURCES}) + target_link_libraries(${TARGET}.generator PRIVATE Halide::Generator ${ARG_LINK_LIBRARIES}) + + # TODO: this should work (and would be preferred to the code above) + # but CMake fails with "targets not yet defined"; investigate. + # add_halide_generator(${TARGET}.generator + # SOURCES ${ARG_SOURCES}) + + # Run the Generator to produce a static library of AOT code, + # plus the 'python_extension' code necessary to produce a useful + # AOT Extention for Python: + add_halide_library(aot_${TARGET} + FROM ${TARGET}.generator + GENERATOR ${ARG_GENERATOR} + FUNCTION_NAME ${ARG_FUNCTION_NAME} + PYTHON_EXTENSION ${TARGET}.py.cpp + FEATURES ${ARG_FEATURES} + PARAMS ${ARG_PARAMS} + TARGETS cmake) + + # Take the native-code output of the Generator, add the Python-Extension + # code (to make it callable from Python), and build it into the AOT Extension we need. + Python3_add_library(${TARGET} MODULE ${${TARGET}.py.cpp}) + target_link_libraries(${TARGET} PRIVATE aot_${TARGET}) + set_target_properties(${TARGET} PROPERTIES OUTPUT_NAME ${ARG_GENERATOR}) # Python3_add_library adds target info to name. + target_export_single_symbol(${TARGET} ${ARG_FUNCTION_NAME}) +endfunction() + +function(add_python_stub_extension TARGET) + set(options) + set(oneValueArgs GENERATOR MODULE) + set(multiValueArgs SOURCES LINK_LIBRARIES) + cmake_parse_arguments(ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + if (NOT ARG_GENERATOR) + set(ARG_GENERATOR "${TARGET}") + endif () + + if (NOT ARG_MODULE) + set(ARG_MODULE "${TARGET}_stub") + endif () + + # Produce a Stub Extension for the same Generator: + # Compiling PyStub.cpp, then linking with the generator's .o file, PyStubImpl.o, + # plus the same libHalide being used by halide.so. + # + # Note that we set HALIDE_PYSTUB_MODULE_NAME to $*_stub (e.g. foo_stub) but + # set HALIDE_PYSTUB_GENERATOR_NAME to the unadorned name of the Generator. + Python3_add_library(${TARGET} MODULE ${_STUB_DIR}/PyStub.cpp ${ARG_SOURCES}) + set_target_properties(${TARGET} PROPERTIES + CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN ON + POSITION_INDEPENDENT_CODE ON) + target_compile_definitions(${TARGET} PRIVATE + "HALIDE_PYSTUB_GENERATOR_NAME=${ARG_GENERATOR}" + "HALIDE_PYSTUB_MODULE_NAME=${ARG_MODULE}") + target_link_libraries(${TARGET} PRIVATE Halide::PyStubs ${ARG_LINK_LIBRARIES}) + set_target_properties(${TARGET} PROPERTIES OUTPUT_NAME ${ARG_MODULE}) + target_export_single_symbol(${TARGET} ${ARG_MODULE}) +endfunction() diff --git a/cmake/TargetExportScript.cmake b/cmake/TargetExportScript.cmake index cbb980baa129..8555869e07b9 100644 --- a/cmake/TargetExportScript.cmake +++ b/cmake/TargetExportScript.cmake @@ -10,7 +10,8 @@ function(target_export_script TARGET) cmake_parse_arguments(ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) get_property(target_type TARGET ${TARGET} PROPERTY TYPE) - if (NOT target_type STREQUAL "SHARED_LIBRARY") + if (NOT target_type STREQUAL "SHARED_LIBRARY" AND NOT target_type STREQUAL "MODULE_LIBRARY") + message(FATAL_ERROR "NOT SHARED ${TARGET} -> ${target_type}") # Linker scripts do nothing on non-shared libraries. return() endif () @@ -35,6 +36,7 @@ function(target_export_script TARGET) if (LINKER_HAS_FLAG_VERSION_SCRIPT) target_link_options(${TARGET} PRIVATE "${version_script}") set_property(TARGET ${TARGET} APPEND PROPERTY LINK_DEPENDS "${ARG_GNU_LD}") + message(STATUS "CASE1 ${TARGET}") return() endif () @@ -47,6 +49,7 @@ function(target_export_script TARGET) if (LINKER_HAS_FLAG_EXPORTED_SYMBOLS_LIST) target_link_options(${TARGET} PRIVATE "${exported_symbols_list}") set_property(TARGET ${TARGET} APPEND PROPERTY LINK_DEPENDS "${ARG_APPLE_LD}") + message(STATUS "CASE2 ${TARGET}") return() endif () diff --git a/python_bindings/CMakeLists.txt b/python_bindings/CMakeLists.txt index 75fc03d6354d..188451560749 100644 --- a/python_bindings/CMakeLists.txt +++ b/python_bindings/CMakeLists.txt @@ -19,7 +19,7 @@ endif () ## add_subdirectory(src) -include(stub/CMakeLists.txt) +add_subdirectory(stub) option(WITH_TEST_PYTHON "Build Python tests" ON) if (WITH_TESTS AND WITH_TEST_PYTHON) diff --git a/python_bindings/Makefile b/python_bindings/Makefile index 6e263f2a4d20..ba2eaacdaaa9 100644 --- a/python_bindings/Makefile +++ b/python_bindings/Makefile @@ -125,17 +125,17 @@ $(BIN)/runtime.a: $(BIN)/runtime.generator # point we want. (If we don't do this we can have interesting failures # when loading multiple of these Python extensions in the same space.) ifeq ($(UNAME), Darwin) -$(BIN)/%.ldscript: correctness/generators/ext.ldscript.apple.in +$(BIN)/%.ldscript: $(ROOT_DIR)/stub/ext.ldscript.apple.in @echo Building $@... @mkdir -p $(@D) - @cat $< | sed 's/$${GEN}/$*/' > $@ + @cat $< | sed 's/$${SYMBOL}/$*/' > $@ PYEXT_LDSCRIPT_FLAG = -Wl,-exported_symbols_list %LDSCRIPT% else # Assume Desktop Linux -$(BIN)/%.ldscript: correctness/generators/ext.ldscript.linux.in +$(BIN)/%.ldscript: $(ROOT_DIR)/stub/ext.ldscript.linux.in @echo Building $@... @mkdir -p $(@D) - @cat $< | sed 's/$${GEN}/$*/' > $@ + @cat $< | sed 's/$${SYMBOL}/$*/' > $@ PYEXT_LDSCRIPT_FLAG = -Wl,--version-script=%LDSCRIPT% endif diff --git a/python_bindings/correctness/generators/CMakeLists.txt b/python_bindings/correctness/generators/CMakeLists.txt index e0b0a8e8456d..95c535c3c926 100644 --- a/python_bindings/correctness/generators/CMakeLists.txt +++ b/python_bindings/correctness/generators/CMakeLists.txt @@ -1,29 +1,4 @@ -include(TargetExportScript) - -# There are two sorts of Python Extensions that we can produce for a Halide Generator -# written in C++: -# -# - One that is essentially the 'native code' output of a Generator, wrapped with enough CPython -# glue code to make it callable from Python. This is analogous to the usual Generator output -# when building a C++ codebase, and is the usual mode used for distribution of final product; -# these correspond to 'ahead-of-time' (AOT) code generation. The resulting code has no dependency -# on libHalide. We'll refer to this sort of extension as an "AOT extension". -# -# - One that essentially *the Generator itself*, wrapped in CPython glue code to make it callable -# from Python at Halide compilation time. This is analogous to the (rarely used) GeneratorStub -# code that can be used to compose multiple Generators together. The resulting extension *does* -# depend on libHalide, and can be used in either JIT or AOT mode for compilation. -# We'll refer to this sort of extension as a "Stub extension". -# -# For testing purposes here, we don't bother using distutils/setuptools to produce a properly-packaged -# Python extension; rather, we simply produce a .so file with the correct name exported, and ensure -# it's in the PYTHONPATH when testing. -# -# In our build files here, we build both kinds of extension for every Generator in the generators/ -# directory (even though not all are used). As a simplistic way to distinguish between the two -# sorts of extensions, we use the unadorned Generator name for AOT extensions, and the Generator name -# suffixed with "_stub" for Stub extensions. (TODO: this is unsatisfyingly hackish; better suggestions -# would be welcome.) +include(PythonExtensionHelpers) set(GENERATORS addconstant @@ -50,89 +25,15 @@ set(GENPARAMS_complex set(GENPARAMS_simple func_input.type=uint8) -# function(add_python_aot_extension TARGET) -# set(options) -# set(oneValueArgs GENERATOR MODULE) -# set(multiValueArgs) -# cmake_parse_arguments(ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - -# if (NOT ARG_GENERATOR) -# set(ARG_GENERATOR "${TARGET}") -# endif () - -# if (NOT ARG_MODULE) -# set(ARG_MODULE "${TARGET}_stub") -# endif () - -# Python3_add_library(${TARGET} MODULE ${HALIDE_PYSTUB_CPP_PATH}) -# target_compile_definitions(${TARGET} PRIVATE -# "HALIDE_PYSTUB_GENERATOR_NAME=${ARG_GENERATOR}" -# "HALIDE_PYSTUB_MODULE_NAME=${ARG_MODULE}") -# target_link_libraries(${TARGET} PRIVATE Halide::PyStubs) -# set_target_properties(${TARGET} PROPERTIES OUTPUT_NAME ${ARG_MODULE}) -# endfunction() - foreach (GEN IN LISTS GENERATORS) - # Create the Halide generator executable. (Prepend "Py_" to avoid name - # conflicts with stuff elsewhere in Halide CMake files.) - add_executable(Py_${GEN}.generator ${GEN}_generator.cpp) - target_link_libraries(Py_${GEN}.generator PRIVATE Halide::Generator) - - # TODO: this should work (and would be preferred to the code above) - # but CMake fails with "targets not yet defined"; investigate. - # add_halide_generator(Py_${GEN}.generator - # SOURCES ${GEN}_generator.cpp) - - # Run the Generator to produce a static library of AOT code, - # plus the 'python_extension' code necessary to produce a useful - # AOT Extention for Python: - add_halide_library(aot_${GEN} - FROM Py_${GEN}.generator - GENERATOR ${GEN} - FUNCTION_NAME ${GEN} - PYTHON_EXTENSION ${GEN}.py.cpp - FEATURES ${FEATURES_${GEN}} - PARAMS ${GENPARAMS_${GEN}} - TARGETS cmake) + add_python_aot_extension(py_aot_${GEN} + GENERATOR ${GEN} + FEATURES ${FEATURES_${GEN}} + PARAMS ${GENPARAMS_${GEN}} + SOURCES ${GEN}_generator.cpp) - # Take the native-code output of the Generator, add the Python-Extension - # code (to make it callable from Python), and build it into the AOT Extension we need. - Python3_add_library(py_aot_${GEN} MODULE ${${GEN}.py.cpp}) - target_link_libraries(py_aot_${GEN} PRIVATE aot_${GEN}) - set_target_properties(py_aot_${GEN} PROPERTIES OUTPUT_NAME ${GEN}) # Python3_add_library adds target info to name. - - # Construct linker script that will export *just* the PyInit entry - # point we want. (If we don't do this we can have interesting failures - # when loading multiple of these Python extensions in the same space.) - # - # TODO: How to do this for Windows as well? - configure_file(ext.ldscript.apple.in "${CMAKE_CURRENT_BINARY_DIR}/${GEN}.ldscript.apple") - configure_file(ext.ldscript.linux.in "${CMAKE_CURRENT_BINARY_DIR}/${GEN}.ldscript") - target_export_script( - py_aot_${GEN} - APPLE_LD "${CMAKE_CURRENT_BINARY_DIR}/${GEN}.ldscript.apple" - GNU_LD "${CMAKE_CURRENT_BINARY_DIR}/${GEN}.ldscript" - ) - - # OK, now we want to produce a Stub Extension for the same Generator: - # Compiling PyStub.cpp, then linking with the generator's .o file, PyStubImpl.o, - # plus the same libHalide being used by halide.so. - # - # Note that we set HALIDE_PYSTUB_MODULE_NAME to $*_stub (e.g. foo_stub) but - # set HALIDE_PYSTUB_GENERATOR_NAME to the unadorned name of the Generator. add_python_stub_extension(py_stub_${GEN} SOURCES ${GEN}_generator.cpp GENERATOR ${GEN} MODULE ${GEN}_stub) - - # Same trick with Linker scripts here. - configure_file(ext.ldscript.apple.in "${CMAKE_CURRENT_BINARY_DIR}/${GEN}_stub.ldscript.apple") - configure_file(ext.ldscript.linux.in "${CMAKE_CURRENT_BINARY_DIR}/${GEN}_stub.ldscript") - target_export_script( - py_stub_${GEN} - APPLE_LD "${CMAKE_CURRENT_BINARY_DIR}/${GEN}.ldscript.apple" - GNU_LD "${CMAKE_CURRENT_BINARY_DIR}/${GEN}.ldscript" - ) - - endforeach () diff --git a/python_bindings/correctness/generators/ext.ldscript.apple.in b/python_bindings/correctness/generators/ext.ldscript.apple.in deleted file mode 100644 index ea4e8830c7a2..000000000000 --- a/python_bindings/correctness/generators/ext.ldscript.apple.in +++ /dev/null @@ -1 +0,0 @@ -_PyInit_${GEN} diff --git a/python_bindings/correctness/generators/ext.ldscript.linux.in b/python_bindings/correctness/generators/ext.ldscript.linux.in deleted file mode 100644 index a75f043345af..000000000000 --- a/python_bindings/correctness/generators/ext.ldscript.linux.in +++ /dev/null @@ -1,4 +0,0 @@ -{ -global: PyInit_${GEN}; -local: *; -}; diff --git a/python_bindings/stub/AddPythonStubExtension.cmake b/python_bindings/stub/AddPythonStubExtension.cmake deleted file mode 100644 index 96da764fff05..000000000000 --- a/python_bindings/stub/AddPythonStubExtension.cmake +++ /dev/null @@ -1,27 +0,0 @@ -set(HALIDE_PYSTUB_CPP_PATH ${CMAKE_CURRENT_LIST_DIR}/PyStub.cpp) - -function(add_python_stub_extension TARGET) - set(options) - set(oneValueArgs GENERATOR MODULE) - set(multiValueArgs SOURCES LINK_LIBRARIES) - cmake_parse_arguments(ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - - if (NOT ARG_GENERATOR) - set(ARG_GENERATOR "${TARGET}") - endif () - - if (NOT ARG_MODULE) - set(ARG_MODULE "${TARGET}_stub") - endif () - - Python3_add_library(${TARGET} MODULE ${HALIDE_PYSTUB_CPP_PATH} ${ARG_SOURCES}) - set_target_properties(${TARGET} PROPERTIES - CXX_VISIBILITY_PRESET hidden - VISIBILITY_INLINES_HIDDEN ON - POSITION_INDEPENDENT_CODE ON) - target_compile_definitions(${TARGET} PRIVATE - "HALIDE_PYSTUB_GENERATOR_NAME=${ARG_GENERATOR}" - "HALIDE_PYSTUB_MODULE_NAME=${ARG_MODULE}") - target_link_libraries(${TARGET} PRIVATE Halide::PyStubs ${ARG_LINK_LIBRARIES}) - set_target_properties(${TARGET} PROPERTIES OUTPUT_NAME ${ARG_MODULE}) -endfunction() diff --git a/python_bindings/stub/CMakeLists.txt b/python_bindings/stub/CMakeLists.txt index 90c1f487aca6..a4f69fc044dd 100644 --- a/python_bindings/stub/CMakeLists.txt +++ b/python_bindings/stub/CMakeLists.txt @@ -10,5 +10,3 @@ if (NOT TARGET Halide::PyStubs) VISIBILITY_INLINES_HIDDEN TRUE POSITION_INDEPENDENT_CODE ON) endif () - -include(${CMAKE_CURRENT_LIST_DIR}/AddPythonStubExtension.cmake) diff --git a/python_bindings/stub/ext.ldscript.apple.in b/python_bindings/stub/ext.ldscript.apple.in new file mode 100644 index 000000000000..90695315ccd6 --- /dev/null +++ b/python_bindings/stub/ext.ldscript.apple.in @@ -0,0 +1 @@ +_PyInit_${SYMBOL} diff --git a/python_bindings/stub/ext.ldscript.linux.in b/python_bindings/stub/ext.ldscript.linux.in new file mode 100644 index 000000000000..b426c20b08b8 --- /dev/null +++ b/python_bindings/stub/ext.ldscript.linux.in @@ -0,0 +1,4 @@ +{ +global: PyInit_${SYMBOL}; +local: *; +}; From e7f037d31c183c47d4fe62765d3b9ab5176326f3 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Wed, 30 Mar 2022 18:15:58 -0700 Subject: [PATCH 7/9] Update TargetExportScript.cmake --- cmake/TargetExportScript.cmake | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmake/TargetExportScript.cmake b/cmake/TargetExportScript.cmake index 8555869e07b9..5f5e438dbcd0 100644 --- a/cmake/TargetExportScript.cmake +++ b/cmake/TargetExportScript.cmake @@ -11,7 +11,6 @@ function(target_export_script TARGET) get_property(target_type TARGET ${TARGET} PROPERTY TYPE) if (NOT target_type STREQUAL "SHARED_LIBRARY" AND NOT target_type STREQUAL "MODULE_LIBRARY") - message(FATAL_ERROR "NOT SHARED ${TARGET} -> ${target_type}") # Linker scripts do nothing on non-shared libraries. return() endif () @@ -36,7 +35,6 @@ function(target_export_script TARGET) if (LINKER_HAS_FLAG_VERSION_SCRIPT) target_link_options(${TARGET} PRIVATE "${version_script}") set_property(TARGET ${TARGET} APPEND PROPERTY LINK_DEPENDS "${ARG_GNU_LD}") - message(STATUS "CASE1 ${TARGET}") return() endif () @@ -49,7 +47,6 @@ function(target_export_script TARGET) if (LINKER_HAS_FLAG_EXPORTED_SYMBOLS_LIST) target_link_options(${TARGET} PRIVATE "${exported_symbols_list}") set_property(TARGET ${TARGET} APPEND PROPERTY LINK_DEPENDS "${ARG_APPLE_LD}") - message(STATUS "CASE2 ${TARGET}") return() endif () From bc09e2df69f1c3e8aee1b6d8cbf02a3b62b59f94 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Tue, 5 Apr 2022 15:02:59 -0700 Subject: [PATCH 8/9] Update PythonExtensionHelpers.cmake --- cmake/PythonExtensionHelpers.cmake | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cmake/PythonExtensionHelpers.cmake b/cmake/PythonExtensionHelpers.cmake index 71f50c2ea481..6ac49603b20e 100644 --- a/cmake/PythonExtensionHelpers.cmake +++ b/cmake/PythonExtensionHelpers.cmake @@ -75,9 +75,17 @@ function(add_python_aot_extension TARGET) # Take the native-code output of the Generator, add the Python-Extension # code (to make it callable from Python), and build it into the AOT Extension we need. - Python3_add_library(${TARGET} MODULE ${${TARGET}.py.cpp}) + if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.17) + # Add soabi info (like cpython-310-x86_64-linux-gnu) + # when CMake is new enough to know how to do it. + set(abi_flags WITH_SOABI) + else () + set(abi_flags "") + endif () + + Python3_add_library(${TARGET} MODULE ${abi_flags} ${${TARGET}.py.cpp}) target_link_libraries(${TARGET} PRIVATE aot_${TARGET}) - set_target_properties(${TARGET} PROPERTIES OUTPUT_NAME ${ARG_GENERATOR}) # Python3_add_library adds target info to name. + set_target_properties(${TARGET} PROPERTIES OUTPUT_NAME ${ARG_GENERATOR}) target_export_single_symbol(${TARGET} ${ARG_FUNCTION_NAME}) endfunction() From b14c68579267aed81fb664797f431f53a8f72e8f Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Tue, 5 Apr 2022 16:47:25 -0700 Subject: [PATCH 9/9] PyExtensionGen didn't handle zero-dimensional buffers --- src/PythonExtensionGen.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/PythonExtensionGen.cpp b/src/PythonExtensionGen.cpp index bcfa189acf63..1b762a2912bc 100644 --- a/src/PythonExtensionGen.cpp +++ b/src/PythonExtensionGen.cpp @@ -101,13 +101,15 @@ std::pair print_type(const LoweredArgument *arg) { void PythonExtensionGen::convert_buffer(const string &name, const LoweredArgument *arg) { internal_assert(arg->is_buffer()); - internal_assert(arg->dimensions); + const int dims_to_use = arg->dimensions; + // Always allocate at least 1 halide_dimension_t, even for zero-dimensional buffers + const int dims_to_allocate = std::max(1, dims_to_use); dest << " halide_buffer_t buffer_" << name << ";\n"; - dest << " halide_dimension_t dimensions_" << name << "[" << (int)arg->dimensions << "];\n"; + dest << " halide_dimension_t dimensions_" << name << "[" << dims_to_allocate << "];\n"; dest << " Py_buffer view_" << name << ";\n"; dest << " if (_convert_py_buffer_to_halide("; dest << /*pyobj*/ "py_" << name << ", "; - dest << /*dimensions*/ (int)arg->dimensions << ", "; + dest << /*dimensions*/ (int)dims_to_use << ", "; dest << /*flags*/ (arg->is_output() ? "PyBUF_WRITABLE" : "0") << ", "; dest << /*dim*/ "dimensions_" << name << ", "; dest << /*out*/ "&buffer_" << name << ", "; @@ -164,7 +166,7 @@ __attribute__((unused)) #endif int _convert_py_buffer_to_halide( PyObject* pyobj, int dimensions, int flags, - halide_dimension_t* dim, // array of size `dimensions` + halide_dimension_t* dim, // array of >= size `dimensions` halide_buffer_t* out, Py_buffer &buf, const char* name) { int ret = PyObject_GetBuffer( pyobj, &buf, PyBUF_FORMAT | PyBUF_STRIDED_RO | PyBUF_ANY_CONTIGUOUS | flags);