refactor: drive from original file

Author: henryiii

bin/update_pythons.py

@@ -1,21 +1,21 @@
 #!/usr/bin/env python3
 
+import copy
 import logging
-from itertools import groupby
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import Dict, Optional, Union
 
 import click
 import requests
 import rich
-import tomlkit
-from packaging.specifiers import SpecifierSet
+import toml
+from packaging.specifiers import Specifier, SpecifierSet
 from packaging.version import Version
 from rich.logging import RichHandler
 from rich.syntax import Syntax
 
 from cibuildwheel.extra import InlineArrayDictEncoder
-from cibuildwheel.typing import Final, Literal, PlatformName, TypedDict
+from cibuildwheel.typing import Final, Literal, TypedDict
 
 log = logging.getLogger("cibw")
 
@@ -32,20 +32,20 @@
 
 class ConfigWinCP(TypedDict):
     identifier: str
-    version: Version
+    version: str
     arch: str
 
 
 class ConfigWinPP(TypedDict):
     identifier: str
-    version: Version
+    version: str
     arch: str
     url: str
 
 
 class ConfigMacOS(TypedDict):
     identifier: str
-    version: Version
+    version: str
     url: str
 
 
@@ -66,7 +66,8 @@ def __init__(self, arch_str: ArchStr) -> None:
         ARCH_DICT = {"32": "win32", "64": "win_amd64"}
         PACKAGE_DICT = {"32": "pythonx86", "64": "python"}
 
-        arch = ARCH_DICT[arch_str]
+        self.arch_str = arch_str
+        self.arch = ARCH_DICT[arch_str]
         package = PACKAGE_DICT[arch_str]
 
         response = requests.get(f"{endpoint}{package}/index.json")
@@ -76,21 +77,21 @@ def __init__(self, arch_str: ArchStr) -> None:
         versions = (Version(v) for v in cp_info["versions"])
         self.versions = sorted(v for v in versions if not v.is_devrelease)
 
-    def update_version(self, spec: Specifier) -> Optional[ConfigWinCP]:
+    def update_version_windows(self, spec: Specifier) -> Optional[ConfigWinCP]:
         versions = sorted(v for v in self.versions if spec.contains(v))
         if not all(v.is_prerelease for v in versions):
             versions = [v for v in versions if not v.is_prerelease]
-        log.debug(versions)
+        log.debug(f"Windows {self.arch} {spec} has {', '.join(str(v) for v in versions)}")
 
         if not versions:
             return None
 
         version = versions[-1]
-        identifier = f"cp{version.major}{version.minor}-{ARCH_DICT[arch]}"
+        identifier = f"cp{version.major}{version.minor}-{self.arch}"
         result = ConfigWinCP(
             identifier=identifier,
-            version=version,
-            arch=arch,
+            version=str(version),
+            arch=self.arch_str,
         )
         return result
 
@@ -106,47 +107,56 @@ def __init__(self, arch_str: ArchStr):
             release["pypy_version"] = Version(release["pypy_version"])
             release["python_version"] = Version(release["python_version"])
 
-        self.releases = [r for r in releases if not r["pypy_version"].is_prerelease and r["pypy_version"].is_devrelease]
-        self.arch == arch_str
+        self.releases = [
+            r for r in releases if not r["pypy_version"].is_prerelease and not r["pypy_version"].is_devrelease
+        ]
+        self.arch = arch_str
 
-    def update_version_windows(self, spec: Specifier) -> Optional[ConfigWinCP]:
+    def update_version_windows(self, spec: Specifier) -> ConfigWinCP:
         if self.arch != "32":
             raise RuntimeError("64 bit releases not supported yet on Windows")
 
-        releases = [r for r in releases if spec.contains(r["python_verison"])]
+        releases = [r for r in self.releases if spec.contains(r["python_version"])]
         releases = sorted(releases, key=lambda r: r["pypy_version"])
 
         if not releases:
-            return None
+            raise RuntimeError(f"PyPy Win {self.arch} not found for {spec}! {self.releases}")
 
         release = releases[-1]
         version = release["python_version"]
         identifier = f"pp{version.major}{version.minor}-win32"
+
+        (url,) = [rf["download_url"] for rf in release["files"] if "" in rf["platform"] == "win32"]
+
         return ConfigWinPP(
             identifier=identifier,
-            version=Version(f"{version.major}.{version.minor}"),
+            version=f"{version.major}.{version.minor}",
             arch="32",
-            url=r["download_url"],
+            url=url,
         )
 
-    def update_version_macos(self, spec: Specifier) -> Optional[ConfigMacOS]:
+    def update_version_macos(self, spec: Specifier) -> ConfigMacOS:
         if self.arch != "64":
             raise RuntimeError("Other archs not supported yet on macOS")
 
-        releases = [r for r in releases if spec.contains(r["python_verison"])]
+        releases = [r for r in self.releases if spec.contains(r["python_version"])]
         releases = sorted(releases, key=lambda r: r["pypy_version"])
 
         if not releases:
-            return None
+            raise RuntimeError(f"PyPy macOS {self.arch} not found for {spec}!")
 
         release = releases[-1]
         version = release["python_version"]
-        identifier = f"pp{version.major}{version.minor}-win32"
+        identifier = f"pp{version.major}{version.minor}-macosx_x86_64"
+
+        (url,) = [
+            rf["download_url"] for rf in release["files"] if "" in rf["platform"] == "darwin" and rf["arch"] == "x64"
+        ]
 
         return ConfigMacOS(
             identifier=identifier,
-            version=Version(f"{version.major}.{version.minor}"),
-            url=rf["download_url"],
+            version=f"{version.major}.{version.minor}",
+            url=url,
         )
 
 
@@ -162,36 +172,89 @@ def __init__(self, plat_arch: str, file_ident: str) -> None:
 
         releases_info = response.json()
 
-        # Removing the prefix, Python 3.9 would use: release["name"].removeprefix("Python ")
-        known_versions = {Version(release["name"][7:]): _get_id(release["resource_uri"]) for release in releases_info}
-        self.versions = sorted(v for v in known_versions if not (v.is_prerelease or v.is_devrelease))
+        self.versions_dict: Dict[Version, int] = {}
+        for release in releases_info:
+            # Removing the prefix, Python 3.9 would use: release["name"].removeprefix("Python ")
+            version = Version(release["name"][7:])
+
+            if not version.is_prerelease and not version.is_devrelease:
+                uri = int(release["resource_uri"].rstrip("/").split("/")[-1])
+                self.versions_dict[version] = uri
 
-    def update_python_macos(self, spec: Specifier) -> Optional[ConfigMacOS]:
+        self.file_ident = file_ident
+        self.plat_arch = plat_arch
+
+    def update_version_macos(self, spec: Specifier) -> Optional[ConfigMacOS]:
 
-        sorted_versions = [v for v in self.versions if spec.contains(v)]
+        sorted_versions = sorted(v for v in self.versions_dict if spec.contains(v))
 
-        for version in reversed(versions):
+        for version in reversed(sorted_versions):
             # Find the first patch version that contains the requested file
-            uri = self.versions[version]
+            uri = self.versions_dict[version]
             response = requests.get(f"https://www.python.org/api/v2/downloads/release_file/?release={uri}")
             response.raise_for_status()
             file_info = response.json()
 
-            canidate_files = [rf["url"] for rf in file_info if file_ident in rf["url"]]
-            if canidate_files:
+            urls = [rf["url"] for rf in file_info if self.file_ident in rf["url"]]
+            if urls:
                 return ConfigMacOS(
-                    identifier=f"cp{version.major}{version.minor}-{plat_arch}",
-                    version=version,
-                    url=canidate_files[0],
+                    identifier=f"cp{version.major}{version.minor}-{self.plat_arch}",
+                    version=f"{version.major}.{version.minor}",
+                    url=urls[0],
                 )
 
         return None
 
 
+class AllVersions:
+    def __init__(self) -> None:
+        self.windows_32 = WindowsVersions("32")
+        self.windows_64 = WindowsVersions("64")
+        self.windows_pypy = PyPyVersions("32")
+
+        self.macos_6 = CPythonVersions(plat_arch="macosx_x86_64", file_ident="macosx10.6.pkg")
+        self.macos_9 = CPythonVersions(plat_arch="macosx_x86_64", file_ident="macosx10.9.pkg")
+        self.macos_u2 = CPythonVersions(
+            plat_arch="macosx_universal2",
+            file_ident="macos11.0.pkg",
+        )
+        self.macos_pypy = PyPyVersions("64")
+
+    def update_config(self, config: Dict[str, str]) -> None:
+        identifier = config["identifier"]
+        version = Version(config["version"])
+        spec = Specifier(f"=={version.major}.{version.minor}.*")
+        log.info(f"Reading in '{identifier}' -> {spec} @ {version}")
+        orig_config = copy.copy(config)
+        config_update: Optional[AnyConfig]
+
+        if "macosx_x86_64" in identifier:
+            if identifier.startswith("pp"):
+                config_update = self.macos_pypy.update_version_macos(spec)
+            else:
+                config_update = self.macos_9.update_version_macos(spec) or self.macos_6.update_version_macos(spec)
+            assert config_update is not None, f"MacOS {spec} not found!"
+            config.update(**config_update)
+        elif "win32" in identifier:
+            if identifier.startswith("pp"):
+                config.update(**self.windows_pypy.update_version_windows(spec))
+            else:
+                config_update = self.windows_32.update_version_windows(spec)
+                if config_update:
+                    config.update(**config_update)
+        elif "win_amd64" in identifier:
+            config_update = self.windows_64.update_version_windows(spec)
+            if config_update:
+                config.update(**config_update)
+
+        if config != orig_config:
+            log.info(f"  Updated {orig_config} to {config}")
+
+
 @click.command()
 @click.option("--inplace", is_flag=True)
 @click.option("--level", default="INFO", type=click.Choice(["INFO", "DEBUG", "TRACE"], case_sensitive=False))
-def update_pythons(inplace: bool, prereleases: bool, level: str) -> None:
+def update_pythons(inplace: bool, level: str) -> None:
 
     logging.basicConfig(
         level="INFO",
@@ -201,34 +264,14 @@ def update_pythons(inplace: bool, prereleases: bool, level: str) -> None:
     )
     log.setLevel(level)
 
-    windows_32 = WindowsVersions("32")
-    windows_64 = WindowsVersions("64")
-    windows_pypy = PyPyVersions("32")
-
-    macos_6 = CPythonVersions(plat_arch="macosx_x86_64", file_ident="macosx10.6.pkg")
-
-    macos_9 = CPythonVersions(plat_arch="macosx_x86_64", file_ident="macosx10.9.pkg")
-
-    macos_u2 = CPythonVersions(
-        plat_arch="macosx_universal2",
-        file_ident="macos11.0.pkg",
-    )
-
-    macos_pypy = PyPyVersions("64")
-
+    all_versions = AllVersions()
     configs = toml.load(RESOURCES_DIR / "build-platforms.toml")
 
     for config in configs["windows"]["python_configurations"]:
-        version = Version(config["version"])
-        spec = Specifier(f"=={version.major}.{version.minor}.*")
-        arch = config["arch"]
-        cpython = config["identifier"].startswith("cp")
+        all_versions.update_config(config)
 
     for config in configs["macos"]["python_configurations"]:
-        version = Version(config["version"])
-        spec = Specifier(f"=={version.major}.{version.minor}.*")
-        arch = "64"
-        pypy = config["identifier"].startswith("pp")
+        all_versions.update_config(config)
 
     if inplace:
         with open(RESOURCES_DIR / "build-platforms.toml", "w") as f: