From 8c6c7ac40970d2de2dcab4c5cb983399951b20fb Mon Sep 17 00:00:00 2001
From: Avasam <samuel.06@hotmail.com>
Date: Mon, 26 Sep 2022 20:51:03 -0400
Subject: [PATCH 01/23] Script to run all checks locally

---
 scripts/create_baseline_stubs.py |  9 ++---
 scripts/run_all_checks.py        | 63 ++++++++++++++++++++++++++++++++
 2 files changed, 67 insertions(+), 5 deletions(-)
 create mode 100644 scripts/run_all_checks.py

diff --git a/scripts/create_baseline_stubs.py b/scripts/create_baseline_stubs.py
index 465ae4805e3f..9b4e1060b96a 100755
--- a/scripts/create_baseline_stubs.py
+++ b/scripts/create_baseline_stubs.py
@@ -169,11 +169,10 @@ def main() -> None:
 
     print("\nDone!\n\nSuggested next steps:")
     print(f" 1. Manually review the generated stubs in {stub_dir}")
-    print(f' 2. Run "MYPYPATH={stub_dir} python3 -m mypy.stubtest {package}" to check the stubs against runtime')
-    print(f' 3. Run "mypy {stub_dir}" to check for errors')
-    print(f' 4. Run "black {stub_dir}" and "isort {stub_dir}" (if you\'ve made code changes)')
-    print(f' 5. Run "flake8 {stub_dir}" to check for e.g. unused imports')
-    print(" 6. Commit the changes on a new branch and create a typeshed PR")
+    print(" 2. See CONTRIBUTING.md for all the best practices and tips to improve your stub")
+    print(' 3. Run autofixes and tests locally using "python3 -m ./scripts/run_all_checks.py"')
+    print("    See tests/README.md to run individual tests")
+    print(" 4. Commit the changes on a new branch and create a typeshed PR (don't force-push!)")
 
 
 if __name__ == "__main__":
diff --git a/scripts/run_all_checks.py b/scripts/run_all_checks.py
new file mode 100644
index 000000000000..fc2286209f26
--- /dev/null
+++ b/scripts/run_all_checks.py
@@ -0,0 +1,63 @@
+import json
+import re
+import subprocess
+import sys
+
+# Popular Flake8 extensions not used in Typeshed that cause errors
+_extend_ignore = "CCE,N8,Q"
+
+_STRICTER_CONFIG_FILE = "pyrightconfig.stricter.json"
+
+
+def _parse_jsonc(json_text: str):
+    parsed_json_text = re.sub(r"^\s*?//.*?\n", "", json_text, flags=re.RegexFlag.MULTILINE)
+    parsed_json_text = re.sub(r",([\n\s]*?[\}\]])", r"\1", parsed_json_text)
+    return parsed_json_text
+
+
+def _get_strict_params(stub_path: str):
+    with open(_STRICTER_CONFIG_FILE) as file:
+        data = json.loads(_parse_jsonc(file.read()))
+    if stub_path in data["exclude"]:
+        return []
+    return ["-p", _STRICTER_CONFIG_FILE]
+
+
+if __name__ == "__main__":
+    try:
+        path = sys.argv[1]
+    except IndexError:
+        print("Missing path argument in format <folder>/<stub>")
+        sys.exit(1)
+    path_tokens = path.replace("\\", "/").split("/")
+    assert len(path_tokens) == 2, "Path argument should be in format <folder>/<stub>"
+    folder, stub = path_tokens
+    assert folder in {"stdlib", "stubs"}, "Only the 'stdlib' and 'stubs' folders are supported"
+
+    subprocess.run(["python3", "-m", "black", path])
+    subprocess.run(["python3", "-m", "isort", path])
+    subprocess.run(["python3", "-m", "flake8", path, "--extend-ignore", _extend_ignore])
+
+    subprocess.run(["python3", "tests/check_consistent.py"])
+    subprocess.run(["python3", "tests/check_new_syntax.py"])
+
+    subprocess.run(["python3", "tests/pyright_test.py", path] + _get_strict_params(path))
+
+    # FIXME: Fails to see requires
+    mypy_result = subprocess.run(["python3", "tests/mypy_test.py", path])
+
+    # TODO: Run in WSL if available
+    if sys.platform != "win32":
+        subprocess.run(["python3", "tests/pytype_test.py", path])
+
+    if folder == "stdlib":
+        subprocess.run(["python3", "tests/regr_test.py", "stdlib"])
+    else:
+        subprocess.run(["python3", "tests/regr_test.py", stub])
+
+    # If mypy failed, this will fail without any helpful error
+    if mypy_result.returncode == 0:
+        if folder == "stdlib":
+            subprocess.run(["python3", "tests/stubtest_stdlib.py", stub])
+        else:
+            subprocess.run(["python3", "tests/stubtest_third_party.py", stub])

From da2191ed1426d24add3a1eeeb1ab9ebec03d7620 Mon Sep 17 00:00:00 2001
From: Avasam <samuel.06@hotmail.com>
Date: Mon, 26 Sep 2022 21:05:26 -0400
Subject: [PATCH 02/23] missing return types

---
 scripts/run_all_checks.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/run_all_checks.py b/scripts/run_all_checks.py
index fc2286209f26..de82b75c2586 100644
--- a/scripts/run_all_checks.py
+++ b/scripts/run_all_checks.py
@@ -9,13 +9,13 @@
 _STRICTER_CONFIG_FILE = "pyrightconfig.stricter.json"
 
 
-def _parse_jsonc(json_text: str):
+def _parse_jsonc(json_text: str) -> str:
     parsed_json_text = re.sub(r"^\s*?//.*?\n", "", json_text, flags=re.RegexFlag.MULTILINE)
     parsed_json_text = re.sub(r",([\n\s]*?[\}\]])", r"\1", parsed_json_text)
     return parsed_json_text
 
 
-def _get_strict_params(stub_path: str):
+def _get_strict_params(stub_path: str) -> list[str]:
     with open(_STRICTER_CONFIG_FILE) as file:
         data = json.loads(_parse_jsonc(file.read()))
     if stub_path in data["exclude"]:

From e3d293fa85f89d07e8ce217b754f13268f5f0d03 Mon Sep 17 00:00:00 2001
From: Samuel T <samuel.06@hotmail.com>
Date: Tue, 27 Sep 2022 10:52:40 -0400
Subject: [PATCH 03/23] Apply suggestions from code review

Co-authored-by: Alex Waygood <Alex.Waygood@Gmail.com>
---
 scripts/create_baseline_stubs.py |  6 ++----
 scripts/run_all_checks.py        | 14 ++++++++------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/scripts/create_baseline_stubs.py b/scripts/create_baseline_stubs.py
index 9b4e1060b96a..f335965cce75 100755
--- a/scripts/create_baseline_stubs.py
+++ b/scripts/create_baseline_stubs.py
@@ -169,10 +169,8 @@ def main() -> None:
 
     print("\nDone!\n\nSuggested next steps:")
     print(f" 1. Manually review the generated stubs in {stub_dir}")
-    print(" 2. See CONTRIBUTING.md for all the best practices and tips to improve your stub")
-    print(' 3. Run autofixes and tests locally using "python3 -m ./scripts/run_all_checks.py"')
-    print("    See tests/README.md to run individual tests")
-    print(" 4. Commit the changes on a new branch and create a typeshed PR (don't force-push!)")
+    print(" 2. Optionally run tests and autofixes (see tests/README.md for details")
+    print(" 3. Commit the changes on a new branch and create a typeshed PR (don't force-push!)")
 
 
 if __name__ == "__main__":
diff --git a/scripts/run_all_checks.py b/scripts/run_all_checks.py
index de82b75c2586..bd052cdb807f 100644
--- a/scripts/run_all_checks.py
+++ b/scripts/run_all_checks.py
@@ -10,9 +10,11 @@
 
 
 def _parse_jsonc(json_text: str) -> str:
-    parsed_json_text = re.sub(r"^\s*?//.*?\n", "", json_text, flags=re.RegexFlag.MULTILINE)
-    parsed_json_text = re.sub(r",([\n\s]*?[\}\]])", r"\1", parsed_json_text)
-    return parsed_json_text
+    # strip comments from the file
+    lines = [line for line in text.split('\n') if not line.strip().startswith('//')]
+    # strip trailing commas from the file
+    valid_json = re.sub(r",(\s*?[\}\]])", r"\1", "\n".join(lines))
+    return valid_json
 
 
 def _get_strict_params(stub_path: str) -> list[str]:
@@ -29,13 +31,14 @@ def _get_strict_params(stub_path: str) -> list[str]:
     except IndexError:
         print("Missing path argument in format <folder>/<stub>")
         sys.exit(1)
-    path_tokens = path.replace("\\", "/").split("/")
+    path_tokens = Path(path).parts
     assert len(path_tokens) == 2, "Path argument should be in format <folder>/<stub>"
     folder, stub = path_tokens
     assert folder in {"stdlib", "stubs"}, "Only the 'stdlib' and 'stubs' folders are supported"
 
-    subprocess.run(["python3", "-m", "black", path])
+    subprocess.run(["python3", "-m", "pycln", path, "--all"])
     subprocess.run(["python3", "-m", "isort", path])
+    subprocess.run(["python3", "-m", "black", path])
     subprocess.run(["python3", "-m", "flake8", path, "--extend-ignore", _extend_ignore])
 
     subprocess.run(["python3", "tests/check_consistent.py"])
@@ -43,7 +46,6 @@ def _get_strict_params(stub_path: str) -> list[str]:
 
     subprocess.run(["python3", "tests/pyright_test.py", path] + _get_strict_params(path))
 
-    # FIXME: Fails to see requires
     mypy_result = subprocess.run(["python3", "tests/mypy_test.py", path])
 
     # TODO: Run in WSL if available

From 9a0757cf0f2386bebc81b62c48fd727ba7aa0e38 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Tue, 27 Sep 2022 14:54:18 +0000
Subject: [PATCH 04/23] [pre-commit.ci] auto fixes from pre-commit.com hooks

---
 scripts/run_all_checks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/run_all_checks.py b/scripts/run_all_checks.py
index bd052cdb807f..d871d8667683 100644
--- a/scripts/run_all_checks.py
+++ b/scripts/run_all_checks.py
@@ -11,7 +11,7 @@
 
 def _parse_jsonc(json_text: str) -> str:
     # strip comments from the file
-    lines = [line for line in text.split('\n') if not line.strip().startswith('//')]
+    lines = [line for line in text.split("\n") if not line.strip().startswith("//")]
     # strip trailing commas from the file
     valid_json = re.sub(r",(\s*?[\}\]])", r"\1", "\n".join(lines))
     return valid_json

From 850d7faf38c06bfa0314d08209fbc78eef647c74 Mon Sep 17 00:00:00 2001
From: Avasam <samuel.06@hotmail.com>
Date: Tue, 27 Sep 2022 15:54:40 -0400
Subject: [PATCH 05/23] summary

---
 scripts/run_all_checks.py | 81 +++++++++++++++++++++++++++++++--------
 1 file changed, 65 insertions(+), 16 deletions(-)

diff --git a/scripts/run_all_checks.py b/scripts/run_all_checks.py
index bd052cdb807f..808f700a00db 100644
--- a/scripts/run_all_checks.py
+++ b/scripts/run_all_checks.py
@@ -2,16 +2,26 @@
 import re
 import subprocess
 import sys
+from pathlib import Path
 
-# Popular Flake8 extensions not used in Typeshed that cause errors
-_extend_ignore = "CCE,N8,Q"
+try:
+    from termcolor import colored
+except ImportError:
+
+    def colored(text: str, color: str = "") -> str:
+        return text
 
+
+# Popular Flake8 extensions not used in Typeshed that cause errors
 _STRICTER_CONFIG_FILE = "pyrightconfig.stricter.json"
+_SUCCESS = colored("Success", "green")
+_SKIPPED = colored("Skipped", "yellow")
+_FAILED = colored("Failed", "red")
 
 
 def _parse_jsonc(json_text: str) -> str:
     # strip comments from the file
-    lines = [line for line in text.split('\n') if not line.strip().startswith('//')]
+    lines = [line for line in json_text.split("\n") if not line.strip().startswith("//")]
     # strip trailing commas from the file
     valid_json = re.sub(r",(\s*?[\}\]])", r"\1", "\n".join(lines))
     return valid_json
@@ -35,31 +45,70 @@ def _get_strict_params(stub_path: str) -> list[str]:
     assert len(path_tokens) == 2, "Path argument should be in format <folder>/<stub>"
     folder, stub = path_tokens
     assert folder in {"stdlib", "stubs"}, "Only the 'stdlib' and 'stubs' folders are supported"
+    stubtest_result = None
+    pytype_result = None
 
+    # Run formatters first. Order matters.
     subprocess.run(["python3", "-m", "pycln", path, "--all"])
     subprocess.run(["python3", "-m", "isort", path])
     subprocess.run(["python3", "-m", "black", path])
-    subprocess.run(["python3", "-m", "flake8", path, "--extend-ignore", _extend_ignore])
 
-    subprocess.run(["python3", "tests/check_consistent.py"])
-    subprocess.run(["python3", "tests/check_new_syntax.py"])
+    flake8_result = subprocess.run(["python3", "-m", "flake8", path])
 
-    subprocess.run(["python3", "tests/pyright_test.py", path] + _get_strict_params(path))
+    check_consistent_result = subprocess.run(["python3", "tests/check_consistent.py"])
+    check_new_syntax_result = subprocess.run(["python3", "tests/check_new_syntax.py"])
+
+    pyright_result = subprocess.run(["python3", "tests/pyright_test.py", path] + _get_strict_params(path))
 
     mypy_result = subprocess.run(["python3", "tests/mypy_test.py", path])
+    # If mypy failed, stubtest will fail without any helpful error
+    if mypy_result.returncode == 0:
+        if folder == "stdlib":
+            stubtest_result = subprocess.run(["python3", "tests/stubtest_stdlib.py", stub])
+        else:
+            stubtest_result = subprocess.run(["python3", "tests/stubtest_third_party.py", stub])
 
     # TODO: Run in WSL if available
     if sys.platform != "win32":
-        subprocess.run(["python3", "tests/pytype_test.py", path])
+        pytype_result = subprocess.run(["python3", "tests/pytype_test.py", path])
 
     if folder == "stdlib":
-        subprocess.run(["python3", "tests/regr_test.py", "stdlib"])
+        regr_test_result = subprocess.run(["python3", "tests/regr_test.py", "stdlib"], stderr=subprocess.PIPE)
     else:
-        subprocess.run(["python3", "tests/regr_test.py", stub])
+        regr_test_result = subprocess.run(["python3", "tests/regr_test.py", stub], stderr=subprocess.PIPE)
+    print(regr_test_result.stderr.decode())
+    # No test means they all ran successfully (0 out of 0). Very few 3rd part stub have regression tests.
+    regr_test_returncode = 0 if b"No test cases found" in regr_test_result.stderr else regr_test_result.returncode
 
-    # If mypy failed, this will fail without any helpful error
-    if mypy_result.returncode == 0:
-        if folder == "stdlib":
-            subprocess.run(["python3", "tests/stubtest_stdlib.py", stub])
-        else:
-            subprocess.run(["python3", "tests/stubtest_third_party.py", stub])
+    any_failure = (
+        flake8_result.returncode
+        + check_consistent_result.returncode
+        + check_new_syntax_result.returncode
+        + pyright_result.returncode
+        + mypy_result.returncode
+        + (stubtest_result.returncode if stubtest_result else 0)
+        + (pytype_result.returncode if pytype_result else 0)
+        + regr_test_returncode
+        > 0
+    )
+
+    if any_failure:
+        print("One or more tests failed. See above for details.")
+    else:
+        print("All tests passed!")
+    print("flake8:", _SUCCESS if flake8_result.returncode == 0 else _FAILED)
+    print("Check consistent:", _SUCCESS if check_consistent_result.returncode == 0 else _FAILED)
+    print("Check new syntax:", _SUCCESS if check_new_syntax_result.returncode == 0 else _FAILED)
+    print("pyright:", _SUCCESS if pyright_result.returncode == 0 else _FAILED)
+    print("mypy:", _SUCCESS if mypy_result.returncode == 0 else _FAILED)
+    if stubtest_result is not None:
+        print("stubtest:", _SUCCESS if stubtest_result.returncode == 0 else _FAILED)
+    else:
+        print("stubtest:", _SKIPPED)
+    if pytype_result is not None:
+        print("pytype:", _SUCCESS if pytype_result.returncode == 0 else _FAILED)
+    else:
+        print("pytype:", _SKIPPED)
+    print("Regression test:", _SUCCESS if regr_test_returncode == 0 else _FAILED)
+
+    sys.exit(int(any_failure))

From 60f3f86d6aac255b1c572ea7e27d8a7e27d66e94 Mon Sep 17 00:00:00 2001
From: Avasam <samuel.06@hotmail.com>
Date: Tue, 27 Sep 2022 16:13:18 -0400
Subject: [PATCH 06/23] skip pyright if no npx

---
 scripts/run_all_checks.py | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/scripts/run_all_checks.py b/scripts/run_all_checks.py
index 808f700a00db..d47bab321983 100644
--- a/scripts/run_all_checks.py
+++ b/scripts/run_all_checks.py
@@ -39,7 +39,7 @@ def _get_strict_params(stub_path: str) -> list[str]:
     try:
         path = sys.argv[1]
     except IndexError:
-        print("Missing path argument in format <folder>/<stub>")
+        print("Missing path argument in format <folder>/<stub>", file=sys.stderr)
         sys.exit(1)
     path_tokens = Path(path).parts
     assert len(path_tokens) == 2, "Path argument should be in format <folder>/<stub>"
@@ -58,7 +58,14 @@ def _get_strict_params(stub_path: str) -> list[str]:
     check_consistent_result = subprocess.run(["python3", "tests/check_consistent.py"])
     check_new_syntax_result = subprocess.run(["python3", "tests/check_new_syntax.py"])
 
-    pyright_result = subprocess.run(["python3", "tests/pyright_test.py", path] + _get_strict_params(path))
+    pyright_result = subprocess.run(["python3", "tests/pyright_test.py", path] + _get_strict_params(path), stderr=subprocess.PIPE)
+    print(pyright_result.stderr.decode())
+    if b"error running npx" in pyright_result.stderr:
+        pyright_returncode = 0
+        pyright_skipped = True
+    else:
+        pyright_returncode = pyright_result.returncode
+        pyright_skipped = False
 
     mypy_result = subprocess.run(["python3", "tests/mypy_test.py", path])
     # If mypy failed, stubtest will fail without any helpful error
@@ -84,7 +91,7 @@ def _get_strict_params(stub_path: str) -> list[str]:
         flake8_result.returncode
         + check_consistent_result.returncode
         + check_new_syntax_result.returncode
-        + pyright_result.returncode
+        + pyright_returncode
         + mypy_result.returncode
         + (stubtest_result.returncode if stubtest_result else 0)
         + (pytype_result.returncode if pytype_result else 0)
@@ -99,16 +106,19 @@ def _get_strict_params(stub_path: str) -> list[str]:
     print("flake8:", _SUCCESS if flake8_result.returncode == 0 else _FAILED)
     print("Check consistent:", _SUCCESS if check_consistent_result.returncode == 0 else _FAILED)
     print("Check new syntax:", _SUCCESS if check_new_syntax_result.returncode == 0 else _FAILED)
-    print("pyright:", _SUCCESS if pyright_result.returncode == 0 else _FAILED)
-    print("mypy:", _SUCCESS if mypy_result.returncode == 0 else _FAILED)
-    if stubtest_result is not None:
-        print("stubtest:", _SUCCESS if stubtest_result.returncode == 0 else _FAILED)
+    if pyright_skipped:
+        print("pyright:", _SKIPPED)
     else:
+        print("pyright:", _SUCCESS if pyright_returncode == 0 else _FAILED)
+    print("mypy:", _SUCCESS if mypy_result.returncode == 0 else _FAILED)
+    if stubtest_result is None:
         print("stubtest:", _SKIPPED)
-    if pytype_result is not None:
-        print("pytype:", _SUCCESS if pytype_result.returncode == 0 else _FAILED)
     else:
+        print("stubtest:", _SUCCESS if stubtest_result.returncode == 0 else _FAILED)
+    if pytype_result is None:
         print("pytype:", _SKIPPED)
+    else:
+        print("pytype:", _SUCCESS if pytype_result.returncode == 0 else _FAILED)
     print("Regression test:", _SUCCESS if regr_test_returncode == 0 else _FAILED)
 
     sys.exit(int(any_failure))

From 6b0544d55a9180ceab38cb15fe5316b6579dee81 Mon Sep 17 00:00:00 2001
From: Avasam <samuel.06@hotmail.com>
Date: Tue, 27 Sep 2022 16:19:47 -0400
Subject: [PATCH 07/23] mypy type ignore

---
 scripts/run_all_checks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/run_all_checks.py b/scripts/run_all_checks.py
index d47bab321983..22565eddc205 100644
--- a/scripts/run_all_checks.py
+++ b/scripts/run_all_checks.py
@@ -8,7 +8,7 @@
     from termcolor import colored
 except ImportError:
 
-    def colored(text: str, color: str = "") -> str:
+    def colored(text: str, color: str = "") -> str:  # type: ignore[misc]
         return text
 
 

From dc9d4eb4a967ff419970f57195c435633c351bb8 Mon Sep 17 00:00:00 2001
From: Avasam <samuel.06@hotmail.com>
Date: Tue, 27 Sep 2022 16:36:29 -0400
Subject: [PATCH 08/23] "Skipping" message

---
 scripts/run_all_checks.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scripts/run_all_checks.py b/scripts/run_all_checks.py
index 22565eddc205..e9c5b26541b2 100644
--- a/scripts/run_all_checks.py
+++ b/scripts/run_all_checks.py
@@ -74,9 +74,12 @@ def _get_strict_params(stub_path: str) -> list[str]:
             stubtest_result = subprocess.run(["python3", "tests/stubtest_stdlib.py", stub])
         else:
             stubtest_result = subprocess.run(["python3", "tests/stubtest_third_party.py", stub])
+    else:
+        print("Skipping stubtest since mypy failed.")
 
-    # TODO: Run in WSL if available
-    if sys.platform != "win32":
+    if sys.platform == "win32":
+        print("Skipping pytype on Windows. You can run the test with WSL.")
+    else:
         pytype_result = subprocess.run(["python3", "tests/pytype_test.py", path])
 
     if folder == "stdlib":

From 0241da076ce5179536d7039796d113585269822c Mon Sep 17 00:00:00 2001
From: Avasam <samuel.06@hotmail.com>
Date: Tue, 27 Sep 2022 16:56:32 -0400
Subject: [PATCH 09/23] mypy false positive

---
 scripts/run_all_checks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/run_all_checks.py b/scripts/run_all_checks.py
index e9c5b26541b2..d5213d786157 100644
--- a/scripts/run_all_checks.py
+++ b/scripts/run_all_checks.py
@@ -97,7 +97,7 @@ def _get_strict_params(stub_path: str) -> list[str]:
         + pyright_returncode
         + mypy_result.returncode
         + (stubtest_result.returncode if stubtest_result else 0)
-        + (pytype_result.returncode if pytype_result else 0)
+        + (pytype_result.returncode if pytype_result else 0)  # type: ignore[attr-defined]
         + regr_test_returncode
         > 0
     )

From 6ad47a3baf06fc654b65f6055070bf910505a461 Mon Sep 17 00:00:00 2001
From: Avasam <samuel.06@hotmail.com>
Date: Tue, 27 Sep 2022 16:57:42 -0400
Subject: [PATCH 10/23] rename

---
 scripts/{run_all_checks.py => runtests.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename scripts/{run_all_checks.py => runtests.py} (100%)

diff --git a/scripts/run_all_checks.py b/scripts/runtests.py
similarity index 100%
rename from scripts/run_all_checks.py
rename to scripts/runtests.py

From 994173ae5309d7f2ac70ca87b4f54414ae867512 Mon Sep 17 00:00:00 2001
From: Avasam <samuel.06@hotmail.com>
Date: Tue, 27 Sep 2022 17:06:54 -0400
Subject: [PATCH 11/23] explicit CompletedProcess type because mypy

---
 scripts/runtests.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scripts/runtests.py b/scripts/runtests.py
index d5213d786157..128dfba3583a 100644
--- a/scripts/runtests.py
+++ b/scripts/runtests.py
@@ -3,6 +3,7 @@
 import subprocess
 import sys
 from pathlib import Path
+from typing import Optional
 
 try:
     from termcolor import colored
@@ -45,8 +46,8 @@ def _get_strict_params(stub_path: str) -> list[str]:
     assert len(path_tokens) == 2, "Path argument should be in format <folder>/<stub>"
     folder, stub = path_tokens
     assert folder in {"stdlib", "stubs"}, "Only the 'stdlib' and 'stubs' folders are supported"
-    stubtest_result = None
-    pytype_result = None
+    stubtest_result: Optional[subprocess.CompletedProcess[bytes]] = None
+    pytype_result: Optional[subprocess.CompletedProcess[bytes]] = None
 
     # Run formatters first. Order matters.
     subprocess.run(["python3", "-m", "pycln", path, "--all"])
@@ -97,7 +98,7 @@ def _get_strict_params(stub_path: str) -> list[str]:
         + pyright_returncode
         + mypy_result.returncode
         + (stubtest_result.returncode if stubtest_result else 0)
-        + (pytype_result.returncode if pytype_result else 0)  # type: ignore[attr-defined]
+        + (pytype_result.returncode if pytype_result else 0)
         + regr_test_returncode
         > 0
     )

From 36aaf68d973f97213b0adaa7970cb2b4743a430f Mon Sep 17 00:00:00 2001
From: Avasam <samuel.06@hotmail.com>
Date: Wed, 28 Sep 2022 19:40:17 -0400
Subject: [PATCH 12/23] mention in readme

---
 tests/README.md | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tests/README.md b/tests/README.md
index 32e848c0c602..5727db3c4775 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -19,6 +19,15 @@ in the `tests` and `scripts` directories.
 To run the tests, follow the [setup instructions](../CONTRIBUTING.md#preparing-the-environment)
 in the `CONTRIBUTING.md` document. In particular, we recommend running with Python 3.9+.
 
+## Run all tests for a specific stub
+
+Run using:
+```
+(.venv3)$ python3 scripts/runtests.py <stdlib-or-stubs>/<stub-to-test>
+```
+
+This script will run all tests below for a specific stub with a summary of the results. You must provide a single argument which is a path to the stub to test, like so: `stblib/os` or `stubs/requests`.
+
 ## mypy\_test.py
 
 Run using:

From 80bc8c7120b3a46a4806e37b3c61a7ca565d1519 Mon Sep 17 00:00:00 2001
From: Samuel T <samuel.06@hotmail.com>
Date: Wed, 28 Sep 2022 19:43:16 -0400
Subject: [PATCH 13/23] Apply suggestions from code review

Co-authored-by: Alex Waygood <Alex.Waygood@Gmail.com>
---
 scripts/runtests.py | 53 +++++++++++++++++++++++----------------------
 1 file changed, 27 insertions(+), 26 deletions(-)

diff --git a/scripts/runtests.py b/scripts/runtests.py
index 128dfba3583a..f5a5e186b1ea 100644
--- a/scripts/runtests.py
+++ b/scripts/runtests.py
@@ -46,20 +46,20 @@ def _get_strict_params(stub_path: str) -> list[str]:
     assert len(path_tokens) == 2, "Path argument should be in format <folder>/<stub>"
     folder, stub = path_tokens
     assert folder in {"stdlib", "stubs"}, "Only the 'stdlib' and 'stubs' folders are supported"
-    stubtest_result: Optional[subprocess.CompletedProcess[bytes]] = None
-    pytype_result: Optional[subprocess.CompletedProcess[bytes]] = None
+    stubtest_result: subprocess.CompletedProcess[bytes] | None = None
+    pytype_result: subprocess.CompletedProcess[bytes] | None = None
 
     # Run formatters first. Order matters.
-    subprocess.run(["python3", "-m", "pycln", path, "--all"])
-    subprocess.run(["python3", "-m", "isort", path])
-    subprocess.run(["python3", "-m", "black", path])
+    subprocess.run([sys.executable, "-m", "pycln", path, "--all"])
+    subprocess.run([sys.executable, "-m", "isort", path])
+    subprocess.run([sys.executable, "-m", "black", path])
 
-    flake8_result = subprocess.run(["python3", "-m", "flake8", path])
+    flake8_result = subprocess.run([sys.executable, "-m", "flake8", path])
 
-    check_consistent_result = subprocess.run(["python3", "tests/check_consistent.py"])
-    check_new_syntax_result = subprocess.run(["python3", "tests/check_new_syntax.py"])
+    check_consistent_result = subprocess.run([sys.executable, "tests/check_consistent.py"])
+    check_new_syntax_result = subprocess.run([sys.executable, "tests/check_new_syntax.py"])
 
-    pyright_result = subprocess.run(["python3", "tests/pyright_test.py", path] + _get_strict_params(path), stderr=subprocess.PIPE)
+    pyright_result = subprocess.run([sys.executable, "tests/pyright_test.py", path] + _get_strict_params(path), stderr=subprocess.PIPE)
     print(pyright_result.stderr.decode())
     if b"error running npx" in pyright_result.stderr:
         pyright_returncode = 0
@@ -68,39 +68,40 @@ def _get_strict_params(stub_path: str) -> list[str]:
         pyright_returncode = pyright_result.returncode
         pyright_skipped = False
 
-    mypy_result = subprocess.run(["python3", "tests/mypy_test.py", path])
+    mypy_result = subprocess.run([sys.executable, "tests/mypy_test.py", path])
     # If mypy failed, stubtest will fail without any helpful error
     if mypy_result.returncode == 0:
         if folder == "stdlib":
-            stubtest_result = subprocess.run(["python3", "tests/stubtest_stdlib.py", stub])
+            stubtest_result = subprocess.run([sys.executable, "tests/stubtest_stdlib.py", stub])
         else:
-            stubtest_result = subprocess.run(["python3", "tests/stubtest_third_party.py", stub])
+            stubtest_result = subprocess.run([sys.executable, "tests/stubtest_third_party.py", stub])
     else:
         print("Skipping stubtest since mypy failed.")
 
     if sys.platform == "win32":
         print("Skipping pytype on Windows. You can run the test with WSL.")
     else:
-        pytype_result = subprocess.run(["python3", "tests/pytype_test.py", path])
+        pytype_result = subprocess.run([sys.executable, "tests/pytype_test.py", path])
 
     if folder == "stdlib":
-        regr_test_result = subprocess.run(["python3", "tests/regr_test.py", "stdlib"], stderr=subprocess.PIPE)
+        regr_test_result = subprocess.run([sys.executable, "tests/regr_test.py", "stdlib"], stderr=subprocess.PIPE)
     else:
-        regr_test_result = subprocess.run(["python3", "tests/regr_test.py", stub], stderr=subprocess.PIPE)
+        regr_test_result = subprocess.run([sys.executable, "tests/regr_test.py", stub], stderr=subprocess.PIPE)
     print(regr_test_result.stderr.decode())
-    # No test means they all ran successfully (0 out of 0). Very few 3rd part stub have regression tests.
+    # No test means they all ran successfully (0 out of 0). Not all 3rd-party stubs have regression tests.
     regr_test_returncode = 0 if b"No test cases found" in regr_test_result.stderr else regr_test_result.returncode
 
-    any_failure = (
-        flake8_result.returncode
-        + check_consistent_result.returncode
-        + check_new_syntax_result.returncode
-        + pyright_returncode
-        + mypy_result.returncode
-        + (stubtest_result.returncode if stubtest_result else 0)
-        + (pytype_result.returncode if pytype_result else 0)
-        + regr_test_returncode
-        > 0
+    any_failure = any(
+        [
+            flake8_result.returncode,
+            check_consistent_result.returncode,
+            check_new_syntax_result.returncode,
+            pyright_returncode,
+            mypy_result.returncode,
+            getattr(stubtest_result, "returncode", 0),
+            getattr(pytype_result, "returncode", 0),
+            regr_test_returncode
+        ]
     )
 
     if any_failure:

From 169ad68b8998b1878b4c9dd2744ad8a836c51f84 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 28 Sep 2022 23:44:28 +0000
Subject: [PATCH 14/23] [pre-commit.ci] auto fixes from pre-commit.com hooks

---
 scripts/runtests.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scripts/runtests.py b/scripts/runtests.py
index f5a5e186b1ea..6fa7357eb9e0 100644
--- a/scripts/runtests.py
+++ b/scripts/runtests.py
@@ -3,7 +3,6 @@
 import subprocess
 import sys
 from pathlib import Path
-from typing import Optional
 
 try:
     from termcolor import colored
@@ -59,7 +58,9 @@ def _get_strict_params(stub_path: str) -> list[str]:
     check_consistent_result = subprocess.run([sys.executable, "tests/check_consistent.py"])
     check_new_syntax_result = subprocess.run([sys.executable, "tests/check_new_syntax.py"])
 
-    pyright_result = subprocess.run([sys.executable, "tests/pyright_test.py", path] + _get_strict_params(path), stderr=subprocess.PIPE)
+    pyright_result = subprocess.run(
+        [sys.executable, "tests/pyright_test.py", path] + _get_strict_params(path), stderr=subprocess.PIPE
+    )
     print(pyright_result.stderr.decode())
     if b"error running npx" in pyright_result.stderr:
         pyright_returncode = 0
@@ -100,7 +101,7 @@ def _get_strict_params(stub_path: str) -> list[str]:
             mypy_result.returncode,
             getattr(stubtest_result, "returncode", 0),
             getattr(pytype_result, "returncode", 0),
-            regr_test_returncode
+            regr_test_returncode,
         ]
     )
 

From 7aa69d3aa9681dd822392e3697415e56744973ea Mon Sep 17 00:00:00 2001
From: Avasam <samuel.06@hotmail.com>
Date: Wed, 28 Sep 2022 19:47:11 -0400
Subject: [PATCH 15/23] annotations

---
 scripts/runtests.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/scripts/runtests.py b/scripts/runtests.py
index f5a5e186b1ea..4b824a2c6f4b 100644
--- a/scripts/runtests.py
+++ b/scripts/runtests.py
@@ -1,9 +1,11 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
 import json
 import re
 import subprocess
 import sys
 from pathlib import Path
-from typing import Optional
 
 try:
     from termcolor import colored
@@ -59,7 +61,9 @@ def _get_strict_params(stub_path: str) -> list[str]:
     check_consistent_result = subprocess.run([sys.executable, "tests/check_consistent.py"])
     check_new_syntax_result = subprocess.run([sys.executable, "tests/check_new_syntax.py"])
 
-    pyright_result = subprocess.run([sys.executable, "tests/pyright_test.py", path] + _get_strict_params(path), stderr=subprocess.PIPE)
+    pyright_result = subprocess.run(
+        [sys.executable, "tests/pyright_test.py", path] + _get_strict_params(path), stderr=subprocess.PIPE
+    )
     print(pyright_result.stderr.decode())
     if b"error running npx" in pyright_result.stderr:
         pyright_returncode = 0
@@ -100,7 +104,7 @@ def _get_strict_params(stub_path: str) -> list[str]:
             mypy_result.returncode,
             getattr(stubtest_result, "returncode", 0),
             getattr(pytype_result, "returncode", 0),
-            regr_test_returncode
+            regr_test_returncode,
         ]
     )
 

From f02191e7f9762fcb10750b1f4e1fa5782a246fba Mon Sep 17 00:00:00 2001
From: Avasam <samuel.06@hotmail.com>
Date: Wed, 28 Sep 2022 19:50:12 -0400
Subject: [PATCH 16/23] obsolete comment

---
 scripts/runtests.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/runtests.py b/scripts/runtests.py
index 4b824a2c6f4b..3e75a8a61e3a 100644
--- a/scripts/runtests.py
+++ b/scripts/runtests.py
@@ -15,7 +15,6 @@ def colored(text: str, color: str = "") -> str:  # type: ignore[misc]
         return text
 
 
-# Popular Flake8 extensions not used in Typeshed that cause errors
 _STRICTER_CONFIG_FILE = "pyrightconfig.stricter.json"
 _SUCCESS = colored("Success", "green")
 _SKIPPED = colored("Skipped", "yellow")

From 31a3c10042e7211cb226a9946fda359f2d907a93 Mon Sep 17 00:00:00 2001
From: Avasam <samuel.06@hotmail.com>
Date: Thu, 29 Sep 2022 12:58:33 -0400
Subject: [PATCH 17/23] PR comments

---
 scripts/runtests.py | 47 ++++++++++++++++++++++++++++++---------------
 tests/README.md     |  4 +++-
 2 files changed, 35 insertions(+), 16 deletions(-)

diff --git a/scripts/runtests.py b/scripts/runtests.py
index 3e75a8a61e3a..981e8b04b3e1 100644
--- a/scripts/runtests.py
+++ b/scripts/runtests.py
@@ -51,29 +51,41 @@ def _get_strict_params(stub_path: str) -> list[str]:
     pytype_result: subprocess.CompletedProcess[bytes] | None = None
 
     # Run formatters first. Order matters.
+    print("\nRunning pycln...")
     subprocess.run([sys.executable, "-m", "pycln", path, "--all"])
+    print("\nRunning isort...")
     subprocess.run([sys.executable, "-m", "isort", path])
-    subprocess.run([sys.executable, "-m", "black", path])
+    print("\nRunning Black...")
+    black_result = subprocess.run([sys.executable, "-m", "black", path])
+    if black_result.returncode == 123:
+        print("Could not run tests due to an internal error with Black. See above for details.", file=sys.stderr)
+        sys.exit(black_result.returncode)
 
+    print("\nRunning Flake8...")
     flake8_result = subprocess.run([sys.executable, "-m", "flake8", path])
 
+    print("\nRunning check_consistent.py...")
     check_consistent_result = subprocess.run([sys.executable, "tests/check_consistent.py"])
+    print("\nRunning check_new_syntax.py...")
     check_new_syntax_result = subprocess.run([sys.executable, "tests/check_new_syntax.py"])
 
+    print("\nRunning Pyright...")
     pyright_result = subprocess.run(
-        [sys.executable, "tests/pyright_test.py", path] + _get_strict_params(path), stderr=subprocess.PIPE
+        [sys.executable, "tests/pyright_test.py", path] + _get_strict_params(path), stderr=subprocess.PIPE, text=True
     )
-    print(pyright_result.stderr.decode())
-    if b"error running npx" in pyright_result.stderr:
+    print(pyright_result.stderr)
+    if re.match(r"error (runn|find)ing npx", pyright_result.stderr):
         pyright_returncode = 0
         pyright_skipped = True
     else:
         pyright_returncode = pyright_result.returncode
         pyright_skipped = False
 
+    print("\nRunning mypy...")
     mypy_result = subprocess.run([sys.executable, "tests/mypy_test.py", path])
     # If mypy failed, stubtest will fail without any helpful error
     if mypy_result.returncode == 0:
+        print("\nRunning stubtest...")
         if folder == "stdlib":
             stubtest_result = subprocess.run([sys.executable, "tests/stubtest_stdlib.py", stub])
         else:
@@ -84,15 +96,20 @@ def _get_strict_params(stub_path: str) -> list[str]:
     if sys.platform == "win32":
         print("Skipping pytype on Windows. You can run the test with WSL.")
     else:
+        print("\nRunning pytype...")
         pytype_result = subprocess.run([sys.executable, "tests/pytype_test.py", path])
 
-    if folder == "stdlib":
-        regr_test_result = subprocess.run([sys.executable, "tests/regr_test.py", "stdlib"], stderr=subprocess.PIPE)
-    else:
-        regr_test_result = subprocess.run([sys.executable, "tests/regr_test.py", stub], stderr=subprocess.PIPE)
-    print(regr_test_result.stderr.decode())
+    print("\nRunning regression tests...")
+    regr_test_result = subprocess.run(
+        [sys.executable, "tests/regr_test.py", "stdlib" if folder == "stdlib" else stub], stderr=subprocess.PIPE, text=True
+    )
     # No test means they all ran successfully (0 out of 0). Not all 3rd-party stubs have regression tests.
-    regr_test_returncode = 0 if b"No test cases found" in regr_test_result.stderr else regr_test_result.returncode
+    if "No test cases found" in regr_test_result.stderr:
+        regr_test_returncode = 0
+        print(f"No test cases found for '{stub}'!")
+    else:
+        regr_test_returncode = regr_test_result.returncode
+        print(regr_test_result.stderr)
 
     any_failure = any(
         [
@@ -108,16 +125,16 @@ def _get_strict_params(stub_path: str) -> list[str]:
     )
 
     if any_failure:
-        print("One or more tests failed. See above for details.")
+        print("\nOne or more tests failed. See above for details.")
     else:
-        print("All tests passed!")
-    print("flake8:", _SUCCESS if flake8_result.returncode == 0 else _FAILED)
+        print("\nAll tests passed!")
+    print("Flake8:", _SUCCESS if flake8_result.returncode == 0 else _FAILED)
     print("Check consistent:", _SUCCESS if check_consistent_result.returncode == 0 else _FAILED)
     print("Check new syntax:", _SUCCESS if check_new_syntax_result.returncode == 0 else _FAILED)
     if pyright_skipped:
-        print("pyright:", _SKIPPED)
+        print("Pyright:", _SKIPPED)
     else:
-        print("pyright:", _SUCCESS if pyright_returncode == 0 else _FAILED)
+        print("Pyright:", _SUCCESS if pyright_returncode == 0 else _FAILED)
     print("mypy:", _SUCCESS if mypy_result.returncode == 0 else _FAILED)
     if stubtest_result is None:
         print("stubtest:", _SKIPPED)
diff --git a/tests/README.md b/tests/README.md
index 5727db3c4775..51d8e2aee22e 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -26,7 +26,9 @@ Run using:
 (.venv3)$ python3 scripts/runtests.py <stdlib-or-stubs>/<stub-to-test>
 ```
 
-This script will run all tests below for a specific stub with a summary of the results. You must provide a single argument which is a path to the stub to test, like so: `stblib/os` or `stubs/requests`.
+This script will run all tests below for a specific typeshed directory. A summary of the results will be printed to the terminal.
+
+You must provide a single argument which is a path to the stubs to test, like so: `stdlib/os` or `stubs/requests`.
 
 ## mypy\_test.py
 

From 77a3c9bc01e2af0e4d5a5db6eb692d915f2e02c4 Mon Sep 17 00:00:00 2001
From: Avasam <samuel.06@hotmail.com>
Date: Sat, 1 Oct 2022 16:22:00 -0400
Subject: [PATCH 18/23] Set python version and better output

---
 scripts/runtests.py | 42 +++++++++++++++++++++++++++++-------------
 tests/README.md     |  2 +-
 2 files changed, 30 insertions(+), 14 deletions(-)

diff --git a/scripts/runtests.py b/scripts/runtests.py
index 981e8b04b3e1..81a2465048cf 100644
--- a/scripts/runtests.py
+++ b/scripts/runtests.py
@@ -19,6 +19,9 @@ def colored(text: str, color: str = "") -> str:  # type: ignore[misc]
 _SUCCESS = colored("Success", "green")
 _SKIPPED = colored("Skipped", "yellow")
 _FAILED = colored("Failed", "red")
+# We're using the oldest supported version because it's the most likely to produce errors
+# due to unsupported syntax, feature, or bug in a tool.
+_PYTHON_VERSION = "3.7"
 
 
 def _parse_jsonc(json_text: str) -> str:
@@ -37,7 +40,7 @@ def _get_strict_params(stub_path: str) -> list[str]:
     return ["-p", _STRICTER_CONFIG_FILE]
 
 
-if __name__ == "__main__":
+def main():
     try:
         path = sys.argv[1]
     except IndexError:
@@ -69,20 +72,23 @@ def _get_strict_params(stub_path: str) -> list[str]:
     print("\nRunning check_new_syntax.py...")
     check_new_syntax_result = subprocess.run([sys.executable, "tests/check_new_syntax.py"])
 
-    print("\nRunning Pyright...")
+    print(f"\nRunning Pyright on Python {_PYTHON_VERSION}...")
     pyright_result = subprocess.run(
-        [sys.executable, "tests/pyright_test.py", path] + _get_strict_params(path), stderr=subprocess.PIPE, text=True
+        [sys.executable, "tests/pyright_test.py", path, "--pythonversion", _PYTHON_VERSION] + _get_strict_params(path),
+        stderr=subprocess.PIPE,
+        text=True,
     )
-    print(pyright_result.stderr)
     if re.match(r"error (runn|find)ing npx", pyright_result.stderr):
+        print("\nSkipping Pyright tests: npx is not installed or can't be run!")
         pyright_returncode = 0
         pyright_skipped = True
     else:
+        print(pyright_result.stderr)
         pyright_returncode = pyright_result.returncode
         pyright_skipped = False
 
-    print("\nRunning mypy...")
-    mypy_result = subprocess.run([sys.executable, "tests/mypy_test.py", path])
+    print(f"\nRunning mypy for Python {_PYTHON_VERSION}...")
+    mypy_result = subprocess.run([sys.executable, "tests/mypy_test.py", path, "--python-version", _PYTHON_VERSION])
     # If mypy failed, stubtest will fail without any helpful error
     if mypy_result.returncode == 0:
         print("\nRunning stubtest...")
@@ -91,22 +97,24 @@ def _get_strict_params(stub_path: str) -> list[str]:
         else:
             stubtest_result = subprocess.run([sys.executable, "tests/stubtest_third_party.py", stub])
     else:
-        print("Skipping stubtest since mypy failed.")
+        print("\nSkipping stubtest since mypy failed.")
 
     if sys.platform == "win32":
-        print("Skipping pytype on Windows. You can run the test with WSL.")
+        print("\nSkipping pytype on Windows. You can run the test with WSL.")
     else:
         print("\nRunning pytype...")
         pytype_result = subprocess.run([sys.executable, "tests/pytype_test.py", path])
 
-    print("\nRunning regression tests...")
+    print(f"\nRunning regression tests for Python {_PYTHON_VERSION}...")
     regr_test_result = subprocess.run(
-        [sys.executable, "tests/regr_test.py", "stdlib" if folder == "stdlib" else stub], stderr=subprocess.PIPE, text=True
+        [sys.executable, "tests/regr_test.py", "stdlib" if folder == "stdlib" else stub, "--python-version", _PYTHON_VERSION],
+        stderr=subprocess.PIPE,
+        text=True,
     )
     # No test means they all ran successfully (0 out of 0). Not all 3rd-party stubs have regression tests.
     if "No test cases found" in regr_test_result.stderr:
         regr_test_returncode = 0
-        print(f"No test cases found for '{stub}'!")
+        print(colored(f"\nNo test cases found for '{stub}'!"), "green")
     else:
         regr_test_returncode = regr_test_result.returncode
         print(regr_test_result.stderr)
@@ -125,9 +133,9 @@ def _get_strict_params(stub_path: str) -> list[str]:
     )
 
     if any_failure:
-        print("\nOne or more tests failed. See above for details.")
+        print(colored("\n\n--- TEST SUMMARY: One or more tests failed. See above for details. ---\n", "red"))
     else:
-        print("\nAll tests passed!")
+        print(colored("\n\n--- TEST SUMMARY: All tests passed! ---\n", "green"))
     print("Flake8:", _SUCCESS if flake8_result.returncode == 0 else _FAILED)
     print("Check consistent:", _SUCCESS if check_consistent_result.returncode == 0 else _FAILED)
     print("Check new syntax:", _SUCCESS if check_new_syntax_result.returncode == 0 else _FAILED)
@@ -147,3 +155,11 @@ def _get_strict_params(stub_path: str) -> list[str]:
     print("Regression test:", _SUCCESS if regr_test_returncode == 0 else _FAILED)
 
     sys.exit(int(any_failure))
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except KeyboardInterrupt:
+        print(colored("\nTests aborted due to KeyboardInterrupt!\n", "red"))
+        sys.exit(1)
diff --git a/tests/README.md b/tests/README.md
index 51d8e2aee22e..2e8805432b02 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -26,7 +26,7 @@ Run using:
 (.venv3)$ python3 scripts/runtests.py <stdlib-or-stubs>/<stub-to-test>
 ```
 
-This script will run all tests below for a specific typeshed directory. A summary of the results will be printed to the terminal.
+This script will run all tests below for a specific typeshed directory. If a test supports multiple python versions, the oldest supported by typeshed will be selected. A summary of the results will be printed to the terminal.
 
 You must provide a single argument which is a path to the stubs to test, like so: `stdlib/os` or `stubs/requests`.
 

From e8e1c43f99237e66ca928463fd6cfeced5e4bb8a Mon Sep 17 00:00:00 2001
From: Avasam <samuel.06@hotmail.com>
Date: Sat, 1 Oct 2022 17:01:47 -0400
Subject: [PATCH 19/23] Typed main

---
 scripts/runtests.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/runtests.py b/scripts/runtests.py
index 81a2465048cf..0ee847b93417 100644
--- a/scripts/runtests.py
+++ b/scripts/runtests.py
@@ -40,7 +40,7 @@ def _get_strict_params(stub_path: str) -> list[str]:
     return ["-p", _STRICTER_CONFIG_FILE]
 
 
-def main():
+def main() -> None:
     try:
         path = sys.argv[1]
     except IndexError:

From 024e6968b441185b3cca0cdf7b5eeb377aaea4f9 Mon Sep 17 00:00:00 2001
From: Avasam <samuel.06@hotmail.com>
Date: Sun, 2 Oct 2022 01:46:33 -0400
Subject: [PATCH 20/23] mention venv bug

---
 tests/README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/README.md b/tests/README.md
index 2e8805432b02..b6d2a6e9baf6 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -25,6 +25,7 @@ Run using:
 ```
 (.venv3)$ python3 scripts/runtests.py <stdlib-or-stubs>/<stub-to-test>
 ```
+*Note: Due to [a bug in venv](https://github.com/python/cpython/issues/87915), on Windows you **have to** use `python` as your executable for 3rd party stubtests to work. `python3` will **not**.*
 
 This script will run all tests below for a specific typeshed directory. If a test supports multiple python versions, the oldest supported by typeshed will be selected. A summary of the results will be printed to the terminal.
 
@@ -125,6 +126,7 @@ Run using
 ```
 (.venv3)$ python3 tests/stubtest_third_party.py
 ```
+*Note: Due to [a bug in venv](https://github.com/python/cpython/issues/87915), on Windows you **have to** use `python` as your executable. `python3` will **not** work.*
 
 Similar to `stubtest_stdlib.py`, but tests the third party stubs. By default,
 it checks all third-party stubs, but you can provide the distributions to

From 3ceb0d3c8e025f6000c3ff0c3c6159b53089f2f1 Mon Sep 17 00:00:00 2001
From: Avasam <samuel.06@hotmail.com>
Date: Sun, 2 Oct 2022 15:38:06 -0400
Subject: [PATCH 21/23] output update

---
 scripts/runtests.py | 26 +++++++++++++++++++-------
 tests/README.md     |  2 --
 2 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/scripts/runtests.py b/scripts/runtests.py
index 0ee847b93417..9a6ab15b3e46 100644
--- a/scripts/runtests.py
+++ b/scripts/runtests.py
@@ -79,7 +79,7 @@ def main() -> None:
         text=True,
     )
     if re.match(r"error (runn|find)ing npx", pyright_result.stderr):
-        print("\nSkipping Pyright tests: npx is not installed or can't be run!")
+        print(colored("\nSkipping Pyright tests: npx is not installed or can't be run!", "yellow"))
         pyright_returncode = 0
         pyright_skipped = True
     else:
@@ -90,17 +90,29 @@ def main() -> None:
     print(f"\nRunning mypy for Python {_PYTHON_VERSION}...")
     mypy_result = subprocess.run([sys.executable, "tests/mypy_test.py", path, "--python-version", _PYTHON_VERSION])
     # If mypy failed, stubtest will fail without any helpful error
-    if mypy_result.returncode == 0:
-        print("\nRunning stubtest...")
+    if mypy_result.returncode == 1:
         if folder == "stdlib":
+            print("\nRunning stubtest...")
             stubtest_result = subprocess.run([sys.executable, "tests/stubtest_stdlib.py", stub])
         else:
-            stubtest_result = subprocess.run([sys.executable, "tests/stubtest_third_party.py", stub])
+            run_stubtest_query = (
+                f"\nRun stubtest for {stub!r} (Y/N)?\n\n"
+                "NOTE: Running third-party stubtest involves downloading and executing arbitrary code from PyPI.\n"
+                f"Only run stubtest if you trust the {stub!r} package.\n"
+            )
+            run_stubtest_answer = input(colored(run_stubtest_query, "yellow")).lower()
+            while run_stubtest_answer not in {"yes", "no", "y", "n"}:
+                run_stubtest_answer = input(colored("Invalid response; please try again.\n", "red")).lower()
+            if run_stubtest_answer in {"yes", "y"}:
+                print("\nRunning stubtest.")
+                stubtest_result = subprocess.run([sys.executable, "tests/stubtest_third_party.py", stub])
+            else:
+                print(colored(f"\nSkipping stubtest for {stub!r}...", "yellow"))
     else:
-        print("\nSkipping stubtest since mypy failed.")
+        print(colored("\nSkipping stubtest since mypy failed.", "yellow"))
 
     if sys.platform == "win32":
-        print("\nSkipping pytype on Windows. You can run the test with WSL.")
+        print(colored("\nSkipping pytype on Windows. You can run the test with WSL.", "yellow"))
     else:
         print("\nRunning pytype...")
         pytype_result = subprocess.run([sys.executable, "tests/pytype_test.py", path])
@@ -114,7 +126,7 @@ def main() -> None:
     # No test means they all ran successfully (0 out of 0). Not all 3rd-party stubs have regression tests.
     if "No test cases found" in regr_test_result.stderr:
         regr_test_returncode = 0
-        print(colored(f"\nNo test cases found for '{stub}'!"), "green")
+        print(colored(f"\nNo test cases found for {stub!r}!", "green"))
     else:
         regr_test_returncode = regr_test_result.returncode
         print(regr_test_result.stderr)
diff --git a/tests/README.md b/tests/README.md
index b6d2a6e9baf6..2e8805432b02 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -25,7 +25,6 @@ Run using:
 ```
 (.venv3)$ python3 scripts/runtests.py <stdlib-or-stubs>/<stub-to-test>
 ```
-*Note: Due to [a bug in venv](https://github.com/python/cpython/issues/87915), on Windows you **have to** use `python` as your executable for 3rd party stubtests to work. `python3` will **not**.*
 
 This script will run all tests below for a specific typeshed directory. If a test supports multiple python versions, the oldest supported by typeshed will be selected. A summary of the results will be printed to the terminal.
 
@@ -126,7 +125,6 @@ Run using
 ```
 (.venv3)$ python3 tests/stubtest_third_party.py
 ```
-*Note: Due to [a bug in venv](https://github.com/python/cpython/issues/87915), on Windows you **have to** use `python` as your executable. `python3` will **not** work.*
 
 Similar to `stubtest_stdlib.py`, but tests the third party stubs. By default,
 it checks all third-party stubs, but you can provide the distributions to

From f9f3154381a0b5c452f36b043aee7e68a81bbbb6 Mon Sep 17 00:00:00 2001
From: Avasam <samuel.06@hotmail.com>
Date: Sun, 2 Oct 2022 15:38:45 -0400
Subject: [PATCH 22/23] revert debug change

---
 scripts/runtests.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/runtests.py b/scripts/runtests.py
index 9a6ab15b3e46..4bec814da12a 100644
--- a/scripts/runtests.py
+++ b/scripts/runtests.py
@@ -90,7 +90,7 @@ def main() -> None:
     print(f"\nRunning mypy for Python {_PYTHON_VERSION}...")
     mypy_result = subprocess.run([sys.executable, "tests/mypy_test.py", path, "--python-version", _PYTHON_VERSION])
     # If mypy failed, stubtest will fail without any helpful error
-    if mypy_result.returncode == 1:
+    if mypy_result.returncode == 0:
         if folder == "stdlib":
             print("\nRunning stubtest...")
             stubtest_result = subprocess.run([sys.executable, "tests/stubtest_stdlib.py", stub])

From fad18bfc28420789cf01ec9179ea932634079b4a Mon Sep 17 00:00:00 2001
From: Samuel T <samuel.06@hotmail.com>
Date: Sun, 2 Oct 2022 16:48:37 -0400
Subject: [PATCH 23/23] Update tests/README.md

Co-authored-by: Alex Waygood <Alex.Waygood@Gmail.com>
---
 tests/README.md | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tests/README.md b/tests/README.md
index 2e8805432b02..acbb01738207 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -26,9 +26,12 @@ Run using:
 (.venv3)$ python3 scripts/runtests.py <stdlib-or-stubs>/<stub-to-test>
 ```
 
-This script will run all tests below for a specific typeshed directory. If a test supports multiple python versions, the oldest supported by typeshed will be selected. A summary of the results will be printed to the terminal.
+This script will run all tests below for a specific typeshed directory. If a
+test supports multiple python versions, the oldest supported by typeshed will
+be selected. A summary of the results will be printed to the terminal.
 
-You must provide a single argument which is a path to the stubs to test, like so: `stdlib/os` or `stubs/requests`.
+You must provide a single argument which is a path to the stubs to test, like
+so: `stdlib/os` or `stubs/requests`.
 
 ## mypy\_test.py