feat: Add python stub files (AcademySoftwareFoundation#4692)

Add python stub files for code completion and static analysis.

Fixes
AcademySoftwareFoundation#4682

No new test cases, since there is not any change to the behavior. I do
have an idea for a kind of test that might validate the stubs against
the python unit tests, but I don't know if it's worth it.

---------

Signed-off-by: Chad Dombrova <chadrik@gmail.com>

Author: chadrik

.github/workflows/wheel.yml

@@ -150,8 +150,12 @@ jobs:
           python-version: '3.9'
 
       - name: Build wheels
+        # Note: the version of cibuildwheel should be kept in sync with src/python/stubs/CMakeLists.txt
         uses: pypa/cibuildwheel@d4a2945fcc8d13f20a1b99d461b8e844d5fc6e23 # v2.21.1
         env:
+          # pass GITHUB_ACTIONS through to the build container so that custom
+          # processes can tell they are running in CI.
+          CIBW_ENVIRONMENT_PASS_LINUX: GITHUB_ACTIONS
           CIBW_BUILD: ${{ matrix.python }}
           CIBW_ARCHS: ${{ matrix.arch }}
           CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux }}

INSTALL.md

@@ -395,6 +395,21 @@ the headers, and the CLI tools to a platform-specific, Python-specific location.
 See the [scikit-build-core docs](https://scikit-build-core.readthedocs.io/en/latest/configuration.html#configuring-cmake-arguments-and-defines)
 for more information on customizing and overriding build-tool options and CMake arguments.
 
+This repo contains python type stubs which are generated from `pybind11` signatures.
+The workflow for releasing new stubs is as follows:
+
+- Install [`uv`](https://docs.astral.sh/uv/getting-started/installation/)
+- Run `make pystubs` locally to generate updated stubs in `src/python/stubs/__init__.pyi`
+- Run `make test-pystubs` locally to use mypy to test the stubs against the code in 
+  the python testsuite.
+- Commit the new stubs and push to Github
+- In CI, the stubs will be included in the wheels built by `cibuildwheel`, as defined in `.github/wheel.yml`
+- In CI, one of the `cibuildwheel` Github actions will rebuild the stubs to a 
+  temp location and verify that they match what has been committed to the repo.  
+  This step ensures that if changes to the C++ source code and bindings results 
+  in a change to the stubs, developers are notified of the need to regenerate 
+  the stubs, so that changes can be reviewed and the rules in `generate_stubs.py` 
+  can be updated, if necessary.
 
 Test Images
 -----------

Makefile

@@ -243,6 +243,18 @@ build: config
 	    ${CMAKE} --build . --config ${CMAKE_BUILD_TYPE} \
 	  )
 
+# generate python stubs and add them to the repo
+pystubs: config
+	@ ( cd ${build_dir} ; \
+	    ${CMAKE} --build . --config ${CMAKE_BUILD_TYPE} --target pystubs \
+	  )
+
+# run mypy on python tests to confirm the stubs are working
+test-pystubs: pystubs
+	@ ( uv export --quiet --no-dev --no-build --no-emit-project --no-hashes -o ${build_dir}/requirements.txt ; \
+	    uvx --with-requirements ${build_dir}/requirements.txt mypy==1.15.0 \
+	  )
+
 # 'make install' builds everthing and installs it in 'dist'.
 # Suppress pointless output from docs installation.
 install: build

pyproject.toml

@@ -122,4 +122,31 @@ CXXFLAGS = "-Wno-error=stringop-overflow -Wno-pragmas"
 SKBUILD_CMAKE_BUILD_TYPE = "MinSizeRel"
 
 [tool.cibuildwheel.windows.environment]
-SKBUILD_CMAKE_BUILD_TYPE = "MinSizeRel"
+SKBUILD_CMAKE_BUILD_TYPE = "MinSizeRel"
+
+[[tool.cibuildwheel.overrides]]
+# Trigger the build & validation of the python stubs for certain platforms.
+# The test command acts as a kind of "post-build" callback where it's possible
+# for the stub-generator to import the freshly-built wheel.
+# There are two entry-points which are designed to call generate_stubs.py through
+# this test command:
+# - `make pystubs` is called during local development to generate the
+#   stubs and copy them into the git repo to be committed and reviewed.
+# - in CI, the cibuildwheel action is used to validate that the stubs match what
+#   has been committed to the repo.
+test-requires = "mypy~=1.15.0 stubgenlib~=0.1.0"
+# Note: the python version here must be kept in sync with src/python/stubs/CMakeLists.txt
+select = "cp311-manylinux_*64"
+inherit.test-command = "append"
+test-command = [
+    "python {project}/src/python/stubs/generate_stubs.py --out-path '/output' --validate-path '{project}/src/python/stubs/OpenImageIO/__init__.pyi'",
+]
+
+[tool.mypy]
+files = [
+    "testsuite/python-*/src/",
+]
+mypy_path = [
+    "src/python/stubs",
+]
+check_untyped_defs = true

src/cmake/pythonutils.cmake

@@ -156,7 +156,8 @@ macro (setup_python_module)
              RUNTIME DESTINATION ${PYTHON_SITE_DIR} COMPONENT user
              LIBRARY DESTINATION ${PYTHON_SITE_DIR} COMPONENT user)
 
-    install(FILES __init__.py DESTINATION ${PYTHON_SITE_DIR} COMPONENT user)
+    install (FILES __init__.py stubs/OpenImageIO/__init__.pyi stubs/OpenImageIO/py.typed
+             DESTINATION ${PYTHON_SITE_DIR} COMPONENT user)
 
 endmacro ()
 

src/python/CMakeLists.txt

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # https://github.com/AcademySoftwareFoundation/OpenImageIO
 
-
+add_subdirectory (stubs)
 
 file (GLOB python_srcs *.cpp)
 setup_python_module (TARGET    PyOpenImageIO
@@ -19,3 +19,4 @@ set_target_properties(PyOpenImageIO PROPERTIES
                           UNITY_BUILD_BATCH_SIZE ${UNITY_SMALL_BATCH_SIZE})
 set_source_files_properties(${python_srcs} PROPERTIES
                             UNITY_GROUP PyOpenImageIO)
+

src/python/stubs/CMakeLists.txt

@@ -0,0 +1,27 @@
+
+# Setup the pystub target, which is not built by default.
+
+set (_stub_file "${CMAKE_SOURCE_DIR}/src/python/stubs/OpenImageIO/__init__.pyi")
+
+# Note: the python version must be kept in sync with `[[tool.cibuildwheel.overrides]]` in pyproject.toml.
+# The stubs are generated within a container so the version of python does not need to match
+# the version of python that OpenImageIO is being built against.
+# Note: the version of cibuildwheel should be kept in sync with .github/workflows/wheel.yml
+add_custom_command (COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/src/python/stubs/generate_stubs_local.py
+                        --repo-root ${CMAKE_SOURCE_DIR} --python-version="3.11" --cibuildwheel-version="2.21.1"
+                        --output-dir "${CMAKE_BINARY_DIR}/wheelhouse"
+                    OUTPUT "${CMAKE_BINARY_DIR}/wheelhouse/OpenImageIO/__init__.pyi"
+                    DEPENDS "${CMAKE_SOURCE_DIR}/src/python/stubs/generate_stubs.py"
+                    DEPENDS "${CMAKE_SOURCE_DIR}/src/python/stubs/generate_stubs_local.py"
+                    COMMENT "pystubs: Generating python stubs"
+                    )
+
+add_custom_command (COMMAND ${CMAKE_COMMAND} -E copy
+                        "${CMAKE_BINARY_DIR}/wheelhouse/OpenImageIO/__init__.pyi"
+                        ${_stub_file}
+                    OUTPUT ${_stub_file}
+                    DEPENDS "${CMAKE_BINARY_DIR}/wheelhouse/OpenImageIO/__init__.pyi"
+                    COMMENT "pystubs: Copying generated stubs to source"
+                    )
+
+add_custom_target (pystubs DEPENDS ${_stub_file})