refactor: rewrite and expect install

Author: henryiii

.github/workflows/update-dependencies.yml

@@ -21,7 +21,7 @@ jobs:
     - name: "Run update: dependencies"
       run: python ./bin/update_dependencies.py
     - name: "Run update: python configs"
-      run: python ./bin/update_dependencies.py
+      run: python ./bin/update_pythons.py --inplace
     - name: Create Pull Request
       if: github.ref == 'refs/heads/master'
       uses: peter-evans/create-pull-request@v3

.pre-commit-config.yaml

@@ -21,6 +21,7 @@ repos:
   hooks:
   - id: black
     files: ^bin/update_pythons.py$
+    args: ["--line-length=120"]
 
 - repo: https://github.com/pre-commit/mirrors-mypy
   rev: v0.790

bin/update_pythons.py

@@ -1,57 +1,40 @@
 #!/usr/bin/env python3
 
-import sys
-from collections import defaultdict
 from itertools import groupby
 from pathlib import Path
-from typing import Dict, List, Tuple
+from typing import List
 
+import click
 import requests
 import toml
 from packaging.version import Version
 
-if sys.version_info < (3, 8):
-    from typing_extensions import TypedDict
-else:
-    from typing import TypedDict
-
-# Use pretty printing for debugging
-# from rich import print
-
-
-allow_prerelease = False
-
-
-class InlineArrayDictEncoder(toml.encoder.TomlEncoder):
-    def dump_sections(self, o: dict, sup: str):
-        if all(isinstance(a, list) for a in o.values()):
-            val = ""
-            for k, v in o.items():
-                inner = ",\n  ".join(self.dump_inline_table(d_i).strip() for d_i in v)
-                val += f"{k} = [\n  {inner},\n]\n"
-            return val, self._dict()
-        else:
-            return super().dump_sections(o, sup)
-
+from cibuildwheel.extra import InlineArrayDictEncoder
+from cibuildwheel.typing import PlatformName, TypedDict
 
+# Looking up the dir instead of using utils.resources_dir
+# since we want to write to it.
 DIR = Path(__file__).parent.parent.resolve()
 RESOURCES_DIR = DIR / "cibuildwheel/resources"
 
 
-class ConfigWinCP(TypedDict):
+class AnyConfig(TypedDict):
     identifier: str
-    version: str
+    version: Version
+
+
+class ConfigWinCP(AnyConfig):
     arch: str
 
 
-class ConfigWinPP(TypedDict):
-    identifier: str
-    version: str
+class ConfigWinPP(AnyConfig):
     arch: str
     url: str
 
 
-def get_cpython_windows() -> Dict[str, List[Version]]:
+def get_cpython_windows() -> List[ConfigWinCP]:
+    ARCH_DICT = {"32": "win32", "64": "win_amd64"}
+
     response = requests.get("https://api.nuget.org/v3/index.json")
     response.raise_for_status()
     api_info = response.json()
@@ -60,120 +43,140 @@ def get_cpython_windows() -> Dict[str, List[Version]]:
         if resource["@type"] == "PackageBaseAddress/3.0.0":
             endpoint = resource["@id"]
 
-    cp_versions: Dict[str, List[Version]] = {"64": [], "32": []}
-    for id, package in [("64", "python"), ("32", "pythonx86")]:
+    items: List[ConfigWinCP] = []
+
+    for arch, package in [("64", "python"), ("32", "pythonx86")]:
         response = requests.get(f"{endpoint}{package}/index.json")
         response.raise_for_status()
         cp_info = response.json()
 
         for version_str in cp_info["versions"]:
             version = Version(version_str)
+
             if version.is_devrelease:
                 continue
-            if not allow_prerelease and version.is_prerelease:
-                continue
-            cp_versions[id].append(version)
-        cp_versions[id].sort()
 
-    return cp_versions
+            identifier = f"cp{version.major}{version.minor}-{ARCH_DICT[arch]}"
+
+            items.append(
+                ConfigWinCP(
+                    identifier=identifier,
+                    version=version,
+                    arch=arch,
+                )
+            )
+    return items
 
 
-def get_pypy_windows(
-    plat_arch: str = "win32-x86",
-) -> Dict[str, List[Tuple[Version, str]]]:
+def get_pypy(platform: PlatformName) -> List[AnyConfig]:
 
     response = requests.get("https://downloads.python.org/pypy/versions.json")
     response.raise_for_status()
-    pp_realeases = response.json()
-    pp_versions = defaultdict(list)
+    pp_releases = response.json()
 
-    for pp_realease in pp_realeases:
-        if pp_realease["pypy_version"] == "nightly":
-            continue
-        version = Version(pp_realease["pypy_version"])
-        python_version = Version(pp_realease["python_version"])
-        python_version_str = f"{python_version.major}.{python_version.minor}"
-        url = None
-        for file in pp_realease["files"]:
-            if f"{file['platform']}-{file['arch']}" == plat_arch:
-                url = file["download_url"]
-                break
-        if url:
-            pp_versions[python_version_str].append((version, url))
+    items: List[AnyConfig] = []
 
-    return pp_versions
+    for pp_release in pp_releases:
 
+        if pp_release["pypy_version"] == "nightly":
+            continue
+        pypy_version = Version(pp_release["pypy_version"])
+        if pypy_version.is_prerelease or pypy_version.is_devrelease:
+            continue
 
-# Debugging printout:
-# print(get_cpython_windows())
-# print()
-# print("[bold]Getting PyPy")
-# print(get_pypy_windows())
+        version = Version(pp_release["python_version"])
+
+        for rf in pp_release["files"]:
+            if platform == "windows":
+                if rf["platform"] == "win32" and rf["arch"] == "x86":
+                    identifier = f"pp{version.major}{version.minor}-win32"
+                    items.append(
+                        ConfigWinPP(
+                            identifier=identifier,
+                            version=Version(f"{version.major}.{version.minor}"),
+                            arch="32",
+                            url=rf["download_url"],
+                        )
+                    )
+                    break
 
-ARCH_DICT = {"32": "win32", "64": "win_amd64"}
+    return items
 
 
-def build_ids_cp(in_dict: Dict[str, List[Version]]) -> List[ConfigWinCP]:
-    items: List[ConfigWinCP] = []
-    for arch in in_dict:
-        for minor, grp in groupby(in_dict[arch], lambda v: v.minor):
-            # Filter pre-releases, unless it's all pre-releases
-            if not all(v.is_devrelease for v in grp):
-                grp = filter(lambda v: not v.is_devrelease, grp)
+def sort_and_filter_configs(
+    orig_items: List[AnyConfig],
+    *,
+    prereleases: bool = False,
+) -> List[AnyConfig]:
 
-            version = sorted(grp)[-1]
-            identifier = f"cp3{minor}-{ARCH_DICT[arch]}"
+    items: List[AnyConfig] = []
 
-            items.append(
-                ConfigWinCP(
-                    identifier=identifier,
-                    version=str(version),
-                    arch=arch,
-                )
-            )
+    # Groupby requires pre-grouped input
+    orig_items = sorted(orig_items, key=lambda x: x["identifier"])
 
-    return items
+    for _, grp in groupby(orig_items, lambda x: x["identifier"]):
+        # Never select dev releases
+        choices = list(filter(lambda x: not x["version"].is_devrelease, grp))
 
+        # Filter pre-releases, unless it's all pre-releases
+        if not all(x["version"].is_prerelease for x in grp):
+            choices = list(filter(lambda x: not x["version"].is_prerelease, choices))
 
-def build_ids_pp(in_dict: Dict[str, List[Tuple[Version, str]]]) -> List[ConfigWinPP]:
-    items: List[ConfigWinPP] = []
-    for vers, matches in in_dict.items():
-        vers_id = vers.replace(".", "")
-        if not all(v[0].is_devrelease for v in matches):
-            matches = list(filter(lambda v: not v[0].is_devrelease, matches))
+        # Select the highest choice unless there are none
+        _url = "url"  # Needed for MyPy, see https://github.com/python/mypy/issues/9902
+        choices = sorted(choices, key=lambda x: (x["version"], x.get(_url)))
+        if not choices:
+            continue
+        best_choice = choices[-1]
 
-        version, url = sorted(matches, key=lambda v: v[0])[-1]
-        identifier = f"pp{vers_id}-win32"
+        # Only allow a pre-release if they are all prereleases, and we've asked for them
+        if best_choice["version"].is_prerelease and not prereleases:
+            continue
 
-        items.append(
-            ConfigWinPP(
-                identifier=identifier,
-                version=vers,
-                arch="32",
-                url=url,
-            )
-        )
+        items.append(best_choice)
 
-    return items
+    return sorted(
+        items,
+        key=lambda x: (
+            x["identifier"][:3],
+            x["version"].minor,
+            x["identifier"].split("-")[-1],
+        ),
+    )
 
 
-windows_configs = [
-    *build_ids_cp(get_cpython_windows()),
-    *build_ids_pp(get_pypy_windows()),
+CLASSIC_WINDOWS: List[ConfigWinCP] = [
+    {"identifier": "cp27-win32", "version": Version("2.7.18"), "arch": "32"},
+    {"identifier": "cp27-win_amd64", "version": Version("2.7.18"), "arch": "64"},
 ]
 
-configs = toml.load(RESOURCES_DIR / "build-platforms.toml")
-origpy2 = list(
-    filter(
-        lambda c: c["identifier"].startswith("cp27"),
-        configs["windows"]["python_configurations"],
-    )
-)
 
-items = origpy2 + windows_configs
-new_configs = sorted(items, key=lambda x: (x["identifier"][:3], x["identifier"][3:]))
+@click.command()
+@click.option("--inplace", is_flag=True)
+@click.option("--prereleases", is_flag=True)
+@click.option("--all", is_flag=True)
+def update_pythons(inplace: bool, prereleases: bool, all: bool) -> None:
+    windows_configs: List[AnyConfig] = [
+        *CLASSIC_WINDOWS,
+        *get_cpython_windows(),
+        *get_pypy("windows"),
+    ]
+
+    if not all:
+        windows_configs = sort_and_filter_configs(
+            windows_configs,
+            prereleases=prereleases,
+        )
+
+    configs = toml.load(RESOURCES_DIR / "build-platforms.toml")
+    configs["windows"]["python_configurations"] = windows_configs
+
+    if inplace:
+        with open(RESOURCES_DIR / "build-platforms.toml", "w") as f:
+            toml.dump(configs, f, encoder=InlineArrayDictEncoder())  # type: ignore
+    else:
+        print(toml.dumps(configs, encoder=InlineArrayDictEncoder()))  # type: ignore
 
-configs["windows"]["python_configurations"] = new_configs
 
-with open(RESOURCES_DIR / "build-platforms.toml", "w") as f:
-    toml.dump(configs, f, encoder=InlineArrayDictEncoder())  # type: ignore
+if __name__ == "__main__":
+    update_pythons()

cibuildwheel/extra.py

@@ -0,0 +1,24 @@
+"""
+These are utilities for the `/bin` scripts, not for the `cibuildwheel` program.
+"""
+
+from typing import Any, Dict
+
+import toml.encoder
+from packaging.version import Version
+
+
+class InlineArrayDictEncoder(toml.encoder.TomlEncoder):  # type: ignore
+    def __init__(self) -> None:
+        super().__init__()
+        self.dump_funcs[Version] = lambda v: f'"{v}"'
+
+    def dump_sections(self, o: Dict[str, Any], sup: str) -> Any:
+        if all(isinstance(a, list) for a in o.values()):
+            val = ""
+            for k, v in o.items():
+                inner = ",\n  ".join(self.dump_inline_table(d_i).strip() for d_i in v)
+                val += f"{k} = [\n  {inner},\n]\n"
+            return val, self._dict()
+        else:
+            return super().dump_sections(o, sup)

cibuildwheel/typing.py

@@ -4,9 +4,12 @@
 from typing import TYPE_CHECKING, NoReturn, Set, Union
 
 if sys.version_info < (3, 8):
-    from typing_extensions import Final, Literal
+    from typing_extensions import Final, Literal, TypedDict
 else:
-    from typing import Final, Literal
+    from typing import Final, Literal, TypedDict
+
+
+__all__ = ("Final", "Literal", "TypedDict", "Set", "Union", "PopenBytes", "PathOrStr", "PlatformName", "PLATFORMS", "assert_never")
 
 
 if TYPE_CHECKING: