feat: support macos too, logging output

Author: henryiii

.pre-commit-config.yaml

@@ -29,7 +29,7 @@ repos:
   - id: mypy
     files: ^(cibuildwheel/|test/|bin/projects.py|bin/update_pythons.py|unit_test/)
     pass_filenames: false
-    additional_dependencies: [packaging]
+    additional_dependencies: [packaging, click]
 
 - repo: https://github.com/asottile/pyupgrade
   rev: v2.7.4

bin/update_pythons.py

@@ -1,21 +1,30 @@
 #!/usr/bin/env python3
 
+import logging
 from itertools import groupby
 from pathlib import Path
 from typing import List
 
 import click
 import requests
+import rich
 import toml
+from packaging.specifiers import SpecifierSet
 from packaging.version import Version
+from rich.logging import RichHandler
+from rich.syntax import Syntax
 
 from cibuildwheel.extra import InlineArrayDictEncoder
-from cibuildwheel.typing import PlatformName, TypedDict
+from cibuildwheel.typing import Final, PlatformName, TypedDict
+
+log = logging.getLogger("cibw")
 
 # Looking up the dir instead of using utils.resources_dir
 # since we want to write to it.
-DIR = Path(__file__).parent.parent.resolve()
-RESOURCES_DIR = DIR / "cibuildwheel/resources"
+DIR: Final[Path] = Path(__file__).parent.parent.resolve()
+RESOURCES_DIR: Final[Path] = DIR / "cibuildwheel/resources"
+
+CIBW_SUPPORTED_PYTHONS: Final[SpecifierSet] = SpecifierSet(">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*")
 
 
 class AnyConfig(TypedDict):
@@ -32,7 +41,12 @@ class ConfigWinPP(AnyConfig):
     url: str
 
 
+class ConfigMacOS(AnyConfig):
+    url: str
+
+
 def get_cpython_windows() -> List[ConfigWinCP]:
+    log.info("[bold]Collecting Windows CPython from nuget")
     ARCH_DICT = {"32": "win32", "64": "win_amd64"}
 
     response = requests.get("https://api.nuget.org/v3/index.json")
@@ -65,10 +79,12 @@ def get_cpython_windows() -> List[ConfigWinCP]:
                     arch=arch,
                 )
             )
+            log.debug(items[-1])
     return items
 
 
 def get_pypy(platform: PlatformName) -> List[AnyConfig]:
+    log.info("[bold]Collecting PyPy from python.org")
 
     response = requests.get("https://downloads.python.org/pypy/versions.json")
     response.raise_for_status()
@@ -98,11 +114,71 @@ def get_pypy(platform: PlatformName) -> List[AnyConfig]:
                             url=rf["download_url"],
                         )
                     )
+                    log.debug(items[-1])
+                    break
+            elif platform == "macos":
+                if rf["platform"] == "darwin" and rf["arch"] == "x64":
+                    identifier = f"pp{version.major}{version.minor}-macosx_x86_64"
+                    items.append(
+                        ConfigMacOS(
+                            identifier=identifier,
+                            version=Version(f"{version.major}.{version.minor}"),
+                            url=rf["download_url"],
+                        )
+                    )
+                    log.debug(items[-1])
                     break
 
     return items
 
 
+def _get_id(resource_uri: str) -> int:
+    return int(resource_uri.rstrip("/").split("/")[-1])
+
+
+def get_cpython(
+    plat_arch: str,
+    file_ident: str,
+    versions: SpecifierSet = CIBW_SUPPORTED_PYTHONS,
+) -> List[ConfigMacOS]:
+    log.info(f"[bold]Collecting {plat_arch} CPython from Python.org")
+
+    response = requests.get("https://www.python.org/api/v2/downloads/release/?is_published=true")
+    response.raise_for_status()
+
+    releases_info = response.json()
+    # Removing the prefix, Python 3.9 would use: release["name"].removeprefix("Python ")
+    known_versions = {Version(release["name"][7:]): _get_id(release["resource_uri"]) for release in releases_info}
+
+    items: List[ConfigMacOS] = []
+
+    sorted_versions = sorted((v for v in known_versions if versions.contains(v) and not v.is_prerelease), reverse=True)
+    # Group is a list of sorted patch versions
+    for pair, group in groupby(sorted_versions, lambda x: (x.major, x.minor)):
+        log.info(f"[bold]Working on {pair[0]}.{pair[1]}")
+        # Find the first patch version that contains the requested file
+        for version in group:
+            uri = known_versions[version]
+
+            log.info(f"  Checking {version}")
+            response = requests.get(f"https://www.python.org/api/v2/downloads/release_file/?release={uri}")
+            response.raise_for_status()
+            file_info = response.json()
+
+            canidate_files = [rf["url"] for rf in file_info if file_ident in rf["url"]]
+            if canidate_files:
+                items.append(
+                    ConfigMacOS(
+                        identifier=f"cp{version.major}{version.minor}-{plat_arch}",
+                        version=version,
+                        url=canidate_files[0],
+                    )
+                )
+                log.info("[green]  Found!")
+                break
+    return items
+
+
 def sort_and_filter_configs(
     orig_items: List[AnyConfig],
     *,
@@ -154,28 +230,65 @@ def sort_and_filter_configs(
 @click.command()
 @click.option("--inplace", is_flag=True)
 @click.option("--prereleases", is_flag=True)
-@click.option("--all", is_flag=True)
-def update_pythons(inplace: bool, prereleases: bool, all: bool) -> None:
+@click.option("--level", default="INFO", type=click.Choice(["INFO", "DEBUG", "TRACE"], case_sensitive=False))
+def update_pythons(inplace: bool, prereleases: bool, level: str) -> None:
+
+    logging.basicConfig(
+        level="INFO",
+        format="%(message)s",
+        datefmt="[%X]",
+        handlers=[RichHandler(rich_tracebacks=True, markup=True)],
+    )
+    log.setLevel(level)
+
     windows_configs: List[AnyConfig] = [
         *CLASSIC_WINDOWS,
         *get_cpython_windows(),
         *get_pypy("windows"),
     ]
 
-    if not all:
-        windows_configs = sort_and_filter_configs(
-            windows_configs,
-            prereleases=prereleases,
-        )
+    windows_configs = sort_and_filter_configs(
+        windows_configs,
+        prereleases=prereleases,
+    )
+
+    macos_configs = [
+        *get_cpython(
+            plat_arch="macosx_x86_64",
+            file_ident="macosx10.9.pkg",
+        ),
+        *get_cpython(
+            plat_arch="macosx_x86_64",
+            file_ident="macosx10.6.pkg",
+            versions=SpecifierSet("==3.5.*"),
+        ),
+        *get_pypy("macos"),
+    ]
+
+    # For universal2:
+    #     plat_arch="macosx_universal2",
+    #     file_ident="macos11.0.pkg",
+    #     versions=SpecifierSet(">=3.8"),
+
+    macos_configs = sort_and_filter_configs(
+        macos_configs,
+        prereleases=prereleases,
+    )
+
+    for config in macos_configs:
+        config["version"] = Version("{0.major}.{0.minor}".format(config["version"]))
 
     configs = toml.load(RESOURCES_DIR / "build-platforms.toml")
     configs["windows"]["python_configurations"] = windows_configs
+    configs["macos"]["python_configurations"] = macos_configs
 
     if inplace:
         with open(RESOURCES_DIR / "build-platforms.toml", "w") as f:
             toml.dump(configs, f, encoder=InlineArrayDictEncoder())  # type: ignore
     else:
-        print(toml.dumps(configs, encoder=InlineArrayDictEncoder()))  # type: ignore
+        output = toml.dumps(configs, encoder=InlineArrayDictEncoder())  # type: ignore
+        rich.print(Syntax(output, "toml", theme="ansi_light"))
+        log.info("File not changed, use --inplace flag to update.")
 
 
 if __name__ == "__main__":

setup.cfg

@@ -52,6 +52,7 @@ dev =
     requests
     typing-extensions
     packaging>=20.8
+    rich>=9.6
 
 [options.packages.find]
 include =