Script to run all checks locally

scripts/create_baseline_stubs.py

@@ -169,11 +169,10 @@ def main() -> None:
 
     print("\nDone!\n\nSuggested next steps:")
     print(f" 1. Manually review the generated stubs in {stub_dir}")
-    print(f' 2. Run "MYPYPATH={stub_dir} python3 -m mypy.stubtest {package}" to check the stubs against runtime')
-    print(f' 3. Run "mypy {stub_dir}" to check for errors')
-    print(f' 4. Run "black {stub_dir}" and "isort {stub_dir}" (if you\'ve made code changes)')
-    print(f' 5. Run "flake8 {stub_dir}" to check for e.g. unused imports')
-    print(" 6. Commit the changes on a new branch and create a typeshed PR")
+    print(" 2. See CONTRIBUTING.md for all the best practices and tips to improve your stub")
+    print(' 3. Run autofixes and tests locally using "python3 -m ./scripts/run_all_checks.py"')
+    print("    See tests/README.md to run individual tests")
+    print(" 4. Commit the changes on a new branch and create a typeshed PR (don't force-push!)")
 
 
 if __name__ == "__main__":

scripts/run_all_checks.py

@@ -0,0 +1,63 @@
+import json
+import re
+import subprocess
+import sys
+
+# Popular Flake8 extensions not used in Typeshed that cause errors
+_extend_ignore = "CCE,N8,Q"
+
+_STRICTER_CONFIG_FILE = "pyrightconfig.stricter.json"
+
+
+def _parse_jsonc(json_text: str) -> str:
+    parsed_json_text = re.sub(r"^\s*?//.*?\n", "", json_text, flags=re.RegexFlag.MULTILINE)
+    parsed_json_text = re.sub(r",([\n\s]*?[\}\]])", r"\1", parsed_json_text)
+    return parsed_json_text
+
+
+def _get_strict_params(stub_path: str) -> list[str]:
+    with open(_STRICTER_CONFIG_FILE) as file:
+        data = json.loads(_parse_jsonc(file.read()))
+    if stub_path in data["exclude"]:
+        return []
+    return ["-p", _STRICTER_CONFIG_FILE]
+
+
+if __name__ == "__main__":
+    try:
+        path = sys.argv[1]
+    except IndexError:
+        print("Missing path argument in format <folder>/<stub>")
+        sys.exit(1)
+    path_tokens = path.replace("\\", "/").split("/")
+    assert len(path_tokens) == 2, "Path argument should be in format <folder>/<stub>"
+    folder, stub = path_tokens
+    assert folder in {"stdlib", "stubs"}, "Only the 'stdlib' and 'stubs' folders are supported"
+
+    subprocess.run(["python3", "-m", "black", path])
+    subprocess.run(["python3", "-m", "isort", path])
+    subprocess.run(["python3", "-m", "flake8", path, "--extend-ignore", _extend_ignore])
+
+    subprocess.run(["python3", "tests/check_consistent.py"])
+    subprocess.run(["python3", "tests/check_new_syntax.py"])
+
+    subprocess.run(["python3", "tests/pyright_test.py", path] + _get_strict_params(path))
+
+    # FIXME: Fails to see requires
+    mypy_result = subprocess.run(["python3", "tests/mypy_test.py", path])
+
+    # TODO: Run in WSL if available
+    if sys.platform != "win32":
+        subprocess.run(["python3", "tests/pytype_test.py", path])
+
+    if folder == "stdlib":
+        subprocess.run(["python3", "tests/regr_test.py", "stdlib"])
+    else:
+        subprocess.run(["python3", "tests/regr_test.py", stub])
+
+    # If mypy failed, this will fail without any helpful error
+    if mypy_result.returncode == 0:
+        if folder == "stdlib":
+            subprocess.run(["python3", "tests/stubtest_stdlib.py", stub])
+        else:
+            subprocess.run(["python3", "tests/stubtest_third_party.py", stub])