gh-100238: Use setuptools in peg-generator and reenable tests

Lib/test/support/__init__.py

@@ -1865,15 +1865,16 @@ def missing_compiler_executable(cmd_names=[]):
     missing.
 
     """
-    # TODO (PEP 632): alternate check without using distutils
-    from distutils import ccompiler, sysconfig, spawn, errors
+    from setuptools._distutils import ccompiler, sysconfig, spawn
+    from setuptools import errors
+
     compiler = ccompiler.new_compiler()
     sysconfig.customize_compiler(compiler)
     if compiler.compiler_type == "msvc":
         # MSVC has no executables, so check whether initialization succeeds
         try:
             compiler.initialize()
-        except errors.DistutilsPlatformError:
+        except errors.PlatformError:
             return "msvc"
     for name in compiler.executables:
         if cmd_names and name not in cmd_names:
@@ -2270,6 +2271,39 @@ def requires_venv_with_pip():
     return unittest.skipUnless(ctypes, 'venv: pip requires ctypes')
 
 
+# Context manager that creates a virtual environment, install setuptools and wheel in it
+# and returns the path to the venv directory and the path to the python executable
+@contextlib.contextmanager
+def setup_venv_with_pip_setuptools_wheel(venv_dir):
+    import subprocess
+    from .os_helper import temp_cwd
+
+    with temp_cwd() as temp_dir:
+        # Create virtual environment to get setuptools
+        cmd = [sys.executable, '-X', 'dev', '-m', 'venv', venv_dir]
+        if verbose:
+            print()
+            print('Run:', ' '.join(cmd))
+        subprocess.run(cmd, check=True)
+
+        # Get the Python executable of the venv
+        python_exe = 'python'
+        if sys.executable.endswith('.exe'):
+            python_exe += '.exe'
+        if sys.platform == 'win32':
+            python = os.path.join(venv_dir, 'Scripts', python_exe)
+        else:
+            python = os.path.join(venv_dir, 'bin', python_exe)
+
+        cmd = [python, '-X', 'dev',
+               '-m', 'pip', 'install',
+               findfile('setuptools-67.6.1-py3-none-any.whl'),
+               findfile('wheel-0.40.0-py3-none-any.whl')]
+        subprocess.run(cmd, check=True)
+
+        yield os.path.join(temp_dir, venv_dir), python
+
+
 # True if Python is built with the Py_DEBUG macro defined: if
 # Python is built in debug mode (./configure --with-pydebug).
 Py_DEBUG = hasattr(sys, 'gettotalrefcount')

Lib/test/test_cppext.py

@@ -35,39 +35,20 @@ def test_build_cpp03(self):
     # the test uses venv+pip: skip if it's not available
     @support.requires_venv_with_pip()
     def check_build(self, std_cpp03, extension_name):
-        # Build in a temporary directory
-        with os_helper.temp_cwd():
-            self._check_build(std_cpp03, extension_name)
+        venv_dir = 'env'
+        with support.setup_venv_with_pip_setuptools_wheel(venv_dir) as (_, python):
+            self._check_build(std_cpp03, extension_name, python)
 
-    def _check_build(self, std_cpp03, extension_name):
+    def _check_build(self, std_cpp03, extension_name, python):
         pkg_dir = 'pkg'
         os.mkdir(pkg_dir)
         shutil.copy(SETUP_TESTCPPEXT, os.path.join(pkg_dir, "setup.py"))
 
-        venv_dir = 'env'
-        verbose = support.verbose
-
-        # Create virtual environment to get setuptools
-        cmd = [sys.executable, '-X', 'dev', '-m', 'venv', venv_dir]
-        if verbose:
-            print()
-            print('Run:', ' '.join(cmd))
-        subprocess.run(cmd, check=True)
-
-        # Get the Python executable of the venv
-        python_exe = 'python'
-        if sys.executable.endswith('.exe'):
-            python_exe += '.exe'
-        if MS_WINDOWS:
-            python = os.path.join(venv_dir, 'Scripts', python_exe)
-        else:
-            python = os.path.join(venv_dir, 'bin', python_exe)
-
         def run_cmd(operation, cmd):
             env = os.environ.copy()
             env['CPYTHON_TEST_CPP_STD'] = 'c++03' if std_cpp03 else 'c++11'
             env['CPYTHON_TEST_EXT_NAME'] = extension_name
-            if verbose:
+            if support.verbose:
                 print('Run:', ' '.join(cmd))
                 subprocess.run(cmd, check=True, env=env)
             else:
@@ -81,12 +62,6 @@ def run_cmd(operation, cmd):
                     self.fail(
                         f"{operation} failed with exit code {proc.returncode}")
 
-        cmd = [python, '-X', 'dev',
-               '-m', 'pip', 'install',
-               support.findfile('setuptools-67.6.1-py3-none-any.whl'),
-               support.findfile('wheel-0.40.0-py3-none-any.whl')]
-        run_cmd('Install build dependencies', cmd)
-
         # Build and install the C++ extension
         cmd = [python, '-X', 'dev',
                '-m', 'pip', 'install', '--no-build-isolation',

Lib/test/test_peg_generator/__init__.py

@@ -3,9 +3,6 @@
 from test import support
 from test.support import load_package_tests
 
-# TODO: gh-92584: peg_generator uses distutils which was removed in Python 3.12
-raise unittest.SkipTest("distutils has been removed in Python 3.12")
-
 
 if support.check_sanitizer(address=True, memory=True):
     # bpo-46633: Skip the test because it is too slow when Python is built

Lib/test/test_peg_generator/test_c_parser.py

@@ -1,3 +1,4 @@
+import contextlib
 import sysconfig
 import textwrap
 import unittest
@@ -8,7 +9,7 @@
 
 from test import test_tools
 from test import support
-from test.support import os_helper
+from test.support import os_helper, import_helper
 from test.support.script_helper import assert_python_ok
 
 _py_cflags_nodist = sysconfig.get_config_var("PY_CFLAGS_NODIST")
@@ -88,6 +89,15 @@ def setUpClass(cls):
         cls.library_dir = tempfile.mkdtemp(dir=cls.tmp_base)
         cls.addClassCleanup(shutil.rmtree, cls.library_dir)
 
+        with contextlib.ExitStack() as stack:
+            full_venv_path, _ = stack.enter_context(support.setup_venv_with_pip_setuptools_wheel("venv"))
+            stack.enter_context(import_helper.DirsOnSysPath(os.path.join(full_venv_path,
+                                                    "lib",
+                                                    f"python{sysconfig.get_python_version()}",
+                                                    "site-packages")))
+            cls.addClassCleanup(stack.pop_all().close)
+
+    @support.requires_venv_with_pip()
     def setUp(self):
         self._backup_config_vars = dict(sysconfig._CONFIG_VARS)
         cmd = support.missing_compiler_executable()

Lib/test/test_peg_generator/test_pegen.py

@@ -552,34 +552,34 @@ def test_mutually_left_recursive(self) -> None:
                                 string="D",
                                 start=(1, 0),
                                 end=(1, 1),
-                                line="D A C A E",
+                                line="D A C A E\n",
                             ),
                             TokenInfo(
                                 type=NAME,
                                 string="A",
                                 start=(1, 2),
                                 end=(1, 3),
-                                line="D A C A E",
+                                line="D A C A E\n",
                             ),
                         ],
                         TokenInfo(
                             type=NAME,
                             string="C",
                             start=(1, 4),
                             end=(1, 5),
-                            line="D A C A E",
+                            line="D A C A E\n",
                         ),
                     ],
                     TokenInfo(
                         type=NAME,
                         string="A",
                         start=(1, 6),
                         end=(1, 7),
-                        line="D A C A E",
+                        line="D A C A E\n",
                     ),
                 ],
                 TokenInfo(
-                    type=NAME, string="E", start=(1, 8), end=(1, 9), line="D A C A E"
+                    type=NAME, string="E", start=(1, 8), end=(1, 9), line="D A C A E\n"
                 ),
             ],
         )
@@ -594,22 +594,22 @@ def test_mutually_left_recursive(self) -> None:
                             string="B",
                             start=(1, 0),
                             end=(1, 1),
-                            line="B C A E",
+                            line="B C A E\n",
                         ),
                         TokenInfo(
                             type=NAME,
                             string="C",
                             start=(1, 2),
                             end=(1, 3),
-                            line="B C A E",
+                            line="B C A E\n",
                         ),
                     ],
                     TokenInfo(
-                        type=NAME, string="A", start=(1, 4), end=(1, 5), line="B C A E"
+                        type=NAME, string="A", start=(1, 4), end=(1, 5), line="B C A E\n"
                     ),
                 ],
                 TokenInfo(
-                    type=NAME, string="E", start=(1, 6), end=(1, 7), line="B C A E"
+                    type=NAME, string="E", start=(1, 6), end=(1, 7), line="B C A E\n"
                 ),
             ],
         )
@@ -654,18 +654,18 @@ def test_lookahead(self) -> None:
             node,
             [
                 TokenInfo(
-                    NAME, string="foo", start=(1, 0), end=(1, 3), line="foo = 12 + 12 ."
+                    NAME, string="foo", start=(1, 0), end=(1, 3), line="foo = 12 + 12 .\n"
                 ),
                 TokenInfo(
-                    OP, string="=", start=(1, 4), end=(1, 5), line="foo = 12 + 12 ."
+                    OP, string="=", start=(1, 4), end=(1, 5), line="foo = 12 + 12 .\n"
                 ),
                 [
                     TokenInfo(
                         NUMBER,
                         string="12",
                         start=(1, 6),
                         end=(1, 8),
-                        line="foo = 12 + 12 .",
+                        line="foo = 12 + 12 .\n",
                     ),
                     [
                         [
@@ -674,14 +674,14 @@ def test_lookahead(self) -> None:
                                 string="+",
                                 start=(1, 9),
                                 end=(1, 10),
-                                line="foo = 12 + 12 .",
+                                line="foo = 12 + 12 .\n",
                             ),
                             TokenInfo(
                                 NUMBER,
                                 string="12",
                                 start=(1, 11),
                                 end=(1, 13),
-                                line="foo = 12 + 12 .",
+                                line="foo = 12 + 12 .\n",
                             ),
                         ]
                     ],
@@ -733,9 +733,9 @@ def test_cut(self) -> None:
         self.assertEqual(
             node,
             [
-                TokenInfo(OP, string="(", start=(1, 0), end=(1, 1), line="(1)"),
-                TokenInfo(NUMBER, string="1", start=(1, 1), end=(1, 2), line="(1)"),
-                TokenInfo(OP, string=")", start=(1, 2), end=(1, 3), line="(1)"),
+                TokenInfo(OP, string="(", start=(1, 0), end=(1, 1), line="(1)\n"),
+                TokenInfo(NUMBER, string="1", start=(1, 1), end=(1, 2), line="(1)\n"),
+                TokenInfo(OP, string=")", start=(1, 2), end=(1, 3), line="(1)\n"),
             ],
         )
 
@@ -794,7 +794,7 @@ def test_soft_keyword(self) -> None:
         start:
             | "number" n=NUMBER { eval(n.string) }
             | "string" n=STRING { n.string }
-            | SOFT_KEYWORD l=NAME n=(NUMBER | NAME | STRING) { f"{l.string} = {n.string}"}
+            | SOFT_KEYWORD l=NAME n=(NUMBER | NAME | STRING) { l.string + " = " + n.string }
         """
         parser_class = make_parser(grammar)
         self.assertEqual(parse_string("number 1", parser_class), 1)

Tools/peg_generator/pegen/build.py

@@ -49,16 +49,15 @@ def compile_c_extension(
     static library of the common parser sources (this is useful in case you are
     creating multiple extensions).
     """
-    import distutils.log
-    from distutils.core import Distribution, Extension
-    from distutils.tests.support import fixup_build_ext  # type: ignore
+    import setuptools.logging
 
-    from distutils.ccompiler import new_compiler
-    from distutils.dep_util import newer_group
-    from distutils.sysconfig import customize_compiler
+    from setuptools import Extension, Distribution
+    from setuptools._distutils.dep_util import newer_group
+    from setuptools._distutils.ccompiler import new_compiler
+    from setuptools._distutils.sysconfig import customize_compiler
 
     if verbose:
-        distutils.log.set_threshold(distutils.log.DEBUG)
+        setuptools.logging.set_threshold(setuptools.logging.logging.DEBUG)
 
     source_file_path = pathlib.Path(generated_source_path)
     extension_name = source_file_path.stem
@@ -100,7 +99,6 @@ def compile_c_extension(
     )
     dist = Distribution({"name": extension_name, "ext_modules": [extension]})
     cmd = dist.get_command_obj("build_ext")
-    fixup_build_ext(cmd)
     cmd.build_lib = str(source_file_path.parent)
     cmd.include_dirs = include_dirs
     if build_dir: