[3.9] Refactor scripts in Tools/peg_generator/scripts (GH-20401)

Modules/_peg_parser.c

@@ -80,14 +80,15 @@ _Py_compile_string(PyObject *self, PyObject *args, PyObject *kwds)
 PyObject *
 _Py_parse_string(PyObject *self, PyObject *args, PyObject *kwds)
 {
-    static char *keywords[] = {"string", "filename", "mode", "oldparser", NULL};
+    static char *keywords[] = {"string", "filename", "mode", "oldparser", "ast", NULL};
     char *the_string;
     char *filename = "<string>";
     char *mode_str = "exec";
     int oldparser = 0;
+    int ast = 1;
 
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|ssp", keywords,
-            &the_string, &filename, &mode_str, &oldparser)) {
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|sspp", keywords,
+            &the_string, &filename, &mode_str, &oldparser, &ast)) {
         return NULL;
     }
 
@@ -110,7 +111,14 @@ _Py_parse_string(PyObject *self, PyObject *args, PyObject *kwds)
         return NULL;
     }
 
-    PyObject *result = PyAST_mod2obj(mod);
+    PyObject *result;
+    if (ast) {
+        result = PyAST_mod2obj(mod);
+    }
+    else {
+        Py_INCREF(Py_None);
+        result = Py_None;
+    }
     PyArena_Free(arena);
     return result;
 }

Tools/peg_generator/Makefile

@@ -70,23 +70,21 @@ stats: peg_extension/parse.c data/xxl.py
 time: time_compile
 
 time_compile: venv data/xxl.py
-	$(VENVPYTHON) scripts/benchmark.py --parser=pegen --target=xxl compile
+	$(VENVPYTHON) scripts/benchmark.py --parser=new --target=xxl compile
 
 time_parse: venv data/xxl.py
-	$(VENVPYTHON) scripts/benchmark.py --parser=pegen --target=xxl parse
+	$(VENVPYTHON) scripts/benchmark.py --parser=new --target=xxl parse
 
 time_old: time_old_compile
 
 time_old_compile: venv data/xxl.py
-	$(VENVPYTHON) scripts/benchmark.py --parser=cpython --target=xxl compile
+	$(VENVPYTHON) scripts/benchmark.py --parser=old --target=xxl compile
 
 time_old_parse: venv data/xxl.py
-	$(VENVPYTHON) scripts/benchmark.py --parser=cpython --target=xxl parse
+	$(VENVPYTHON) scripts/benchmark.py --parser=old --target=xxl parse
 
 time_peg_dir: venv
 	$(VENVPYTHON) scripts/test_parse_directory.py \
-		--grammar-file $(GRAMMAR) \
-		--tokens-file $(TOKENS) \
 		-d $(TESTDIR) \
 		$(TESTFLAGS) \
 		--exclude "*/failset/*" \
@@ -95,12 +93,8 @@ time_peg_dir: venv
 
 time_stdlib: $(CPYTHON) venv
 	$(VENVPYTHON) scripts/test_parse_directory.py \
-		--grammar-file $(GRAMMAR) \
-		--tokens-file $(TOKENS) \
 		-d $(CPYTHON) \
 		$(TESTFLAGS) \
-		--exclude "*/test2to3/*" \
-		--exclude "*/test2to3/**/*" \
 		--exclude "*/bad*" \
 		--exclude "*/lib2to3/tests/data/*"
 

Tools/peg_generator/scripts/benchmark.py

@@ -24,7 +24,7 @@
 argparser.add_argument(
     "--parser",
     action="store",
-    choices=["pegen", "cpython"],
+    choices=["new", "old"],
     default="pegen",
     help="Which parser to benchmark (default is pegen)",
 )
@@ -40,7 +40,12 @@
 command_compile = subcommands.add_parser(
     "compile", help="Benchmark parsing and compiling to bytecode"
 )
-command_parse = subcommands.add_parser("parse", help="Benchmark parsing and generating an ast.AST")
+command_parse = subcommands.add_parser(
+    "parse", help="Benchmark parsing and generating an ast.AST"
+)
+command_notree = subcommands.add_parser(
+    "notree", help="Benchmark parsing and dumping the tree"
+)
 
 
 def benchmark(func):
@@ -62,7 +67,7 @@ def wrapper(*args):
 
 @benchmark
 def time_compile(source, parser):
-    if parser == "cpython":
+    if parser == "old":
         return _peg_parser.compile_string(
             source,
             oldparser=True,
@@ -73,32 +78,40 @@ def time_compile(source, parser):
 
 @benchmark
 def time_parse(source, parser):
-    if parser == "cpython":
+    if parser == "old":
         return _peg_parser.parse_string(source, oldparser=True)
     else:
         return _peg_parser.parse_string(source)
 
 
+@benchmark
+def time_notree(source, parser):
+    if parser == "old":
+        return _peg_parser.parse_string(source, oldparser=True, ast=False)
+    else:
+        return _peg_parser.parse_string(source, ast=False)
+
+
 def run_benchmark_xxl(subcommand, parser, source):
     if subcommand == "compile":
         time_compile(source, parser)
     elif subcommand == "parse":
         time_parse(source, parser)
+    elif subcommand == "notree":
+        time_notree(source, parser)
 
 
 def run_benchmark_stdlib(subcommand, parser):
+    modes = {"compile": 2, "parse": 1, "notree": 0}
     for _ in range(3):
         parse_directory(
             "../../Lib",
-            "../../Grammar/python.gram",
-            "../../Grammar/Tokens",
             verbose=False,
             excluded_files=["*/bad*", "*/lib2to3/tests/data/*",],
-            skip_actions=False,
             tree_arg=0,
             short=True,
-            mode=2 if subcommand == "compile" else 1,
-            parser=parser,
+            mode=modes[subcommand],
+            oldparser=(parser == "old"),
         )
 
 

Tools/peg_generator/scripts/grammar_grapher.py

@@ -42,6 +42,13 @@
 )
 
 argparser = argparse.ArgumentParser(prog="graph_grammar", description="Graph a grammar tree",)
+argparser.add_argument(
+    "-s",
+    "--start",
+    choices=["exec", "eval", "single"],
+    default="exec",
+    help="Choose the grammar's start rule (exec, eval or single)",
+)
 argparser.add_argument("grammar_file", help="The grammar file to graph")
 
 
@@ -91,19 +98,15 @@ def main() -> None:
         references[name] = set(references_for_item(rule))
 
     # Flatten the start node if has only a single reference
-    root_node = "start"
-    if start := references["start"]:
-        if len(start) == 1:
-            root_node = list(start)[0]
-            del references["start"]
+    root_node = {"exec": "file", "eval": "eval", "single": "interactive"}[args.start]
 
     print("digraph g1 {")
     print('\toverlap="scale";')  # Force twopi to scale the graph to avoid overlaps
     print(f'\troot="{root_node}";')
-    print(f"\t{root_node} [color=green, shape=circle]")
+    print(f"\t{root_node} [color=green, shape=circle];")
     for name, refs in references.items():
-        if refs:  # Ignore empty sets
-            print(f"\t{name} -> {','.join(refs)};")
+        for ref in refs:
+            print(f"\t{name} -> {ref};")
     print("}")
 
 

Tools/peg_generator/scripts/show_parse.py

@@ -41,7 +41,13 @@
 parser.add_argument(
     "-d", "--diff", action="store_true", help="show diff between grammar and ast (requires -g)"
 )
-parser.add_argument("-g", "--grammar-file", help="grammar to use (default: use the ast module)")
+parser.add_argument(
+    "-p",
+    "--parser",
+    choices=["new", "old"],
+    default="new",
+    help="choose the parser to use"
+)
 parser.add_argument(
     "-m",
     "--multiline",
@@ -84,31 +90,30 @@ def print_parse(source: str, verbose: bool = False) -> None:
 
 def main() -> None:
     args = parser.parse_args()
-    if args.diff and not args.grammar_file:
-        parser.error("-d/--diff requires -g/--grammar-file")
+    new_parser = args.parser == "new"
     if args.multiline:
         sep = "\n"
     else:
         sep = " "
     program = sep.join(args.program)
-    if args.grammar_file:
+    if new_parser:
         tree = _peg_parser.parse_string(program)
 
         if args.diff:
-            a = tree
-            b = _peg_parser.parse_string(program, oldparser=True)
+            a = _peg_parser.parse_string(program, oldparser=True)
+            b = tree
             diff = diff_trees(a, b, args.verbose)
             if diff:
                 for line in diff:
                     print(line)
             else:
                 print("# Trees are the same")
         else:
-            print(f"# Parsed using {args.grammar_file}")
+            print("# Parsed using the new parser")
             print(format_tree(tree, args.verbose))
     else:
         tree = _peg_parser.parse_string(program, oldparser=True)
-        print("# Parse using the old parser")
+        print("# Parsed using the old parser")
         print(format_tree(tree, args.verbose))