Update python versions as part of update_dependencies

Author: mayeut

.github/workflows/update-dependencies.yml

@@ -14,7 +14,7 @@ jobs:
         python-version: 3.9
         architecture: x64
     - name: Install dependencies
-      run: python -m pip install requests pip-tools
+      run: python -m pip install requests pip-tools packaging>=20.8
     - name: Run update
       run: python ./bin/update_dependencies.py
     - name: Create Pull Request

bin/update_dependencies.py

@@ -5,8 +5,9 @@
 import shutil
 import subprocess
 import sys
-from collections import namedtuple
+from collections import namedtuple, defaultdict
 
+import packaging.version
 import requests
 
 os.chdir(os.path.dirname(__file__))
@@ -117,3 +118,238 @@
 
 with open('cibuildwheel/resources/pinned_docker_images.cfg', 'w') as f:
     config.write(f)
+
+
+# update Python on windows
+# CPython uses nugget
+# c.f. https://docs.microsoft.com/en-us/nuget/api/overview
+response = requests.get('https://api.nuget.org/v3/index.json')
+response.raise_for_status()
+api_info = response.json()
+for resource in api_info['resources']:
+    if resource['@type'] == 'PackageBaseAddress/3.0.0':
+        endpoint = resource['@id']
+cp_versions = {'64': [], '32': [] }
+for id, package in [('64', 'python'), ('32', 'pythonx86')]:
+    response = requests.get(f'{endpoint}{package}/index.json')
+    response.raise_for_status()
+    cp_info = response.json()
+    for version_str in cp_info['versions']:
+        version = packaging.version.parse(version_str)
+        if version.is_devrelease:
+            continue
+        cp_versions[id].append(version)
+    cp_versions[id].sort()
+# PyPy is downloaded from https://downloads.python.org/pypy
+response = requests.get('https://downloads.python.org/pypy/versions.json')
+response.raise_for_status()
+pp_realeases = response.json()
+pp_versions = defaultdict(list)
+for pp_realease in pp_realeases:
+    if pp_realease['pypy_version'] == 'nightly':
+        continue
+    version = packaging.version.parse(pp_realease['pypy_version'])
+    python_version = packaging.version.parse(pp_realease['python_version'])
+    python_version = f'{python_version.major}.{python_version.minor}'
+    url = None
+    for file in pp_realease['files']:
+        if f"{file['platform']}-{file['arch']}" == 'win32-x86':
+            url = file['download_url']
+            break
+    if url:
+        pp_versions[python_version].append((version, url))
+
+# load windows.py
+with open('cibuildwheel/windows.py', 'rt') as f:
+    lines = f.readlines()
+# hugly search pattern, package configuration shall probably done otherwise if we want to do this
+for index, line in enumerate(lines):
+    if 'PythonConfiguration' in line and 'url=None' in line and "identifier='cp3" in line:
+        if "arch='32'" in line:
+            id='32'
+        else:
+            id='64'
+        start = line.index("version='") + 9
+        end = line.index("'", start)
+        current_version = packaging.version.parse(line[start:end])
+        new_version = current_version
+        max_version = packaging.version.parse(f'{current_version.major}.{current_version.minor + 1}')
+        allow_prerelease = False
+        if current_version.is_prerelease:
+            release_version = packaging.version.parse(f'{current_version.major}.{current_version.minor}')
+            if release_version in cp_versions[id]:
+                new_version = release_version
+            else:
+                allow_prerelease = True
+                max_version = release_version
+
+        for version in cp_versions[id]:
+            if version.is_prerelease and not allow_prerelease:
+                continue
+            if version > new_version and version < max_version:
+                new_version = version
+        lines[index] = line[:start] + str(new_version) + line[end:]
+    elif 'PythonConfiguration' in line and "identifier='pp" in line:
+        start = line.index("version='") + 9
+        end = line.index("'", start)
+        id = line[start:end]
+        start = line.index("url='") + 5
+        end = line.index("'", start)
+        current_url = line[start:end]
+        _, current_version_str, _ = current_url.split('/')[-1].split('-')
+        current_version = packaging.version.parse(current_version_str)
+        new_version = current_version
+        new_url = current_url
+        max_version = packaging.version.parse(f'{current_version.major}.{current_version.minor + 1}')
+        allow_prerelease = False
+        if current_version.is_prerelease:
+            release_version = packaging.version.parse(f'{current_version.major}.{current_version.minor}')
+            found_release = False
+            for version, url in pp_versions[id]:
+                if release_version == version:
+                    new_version = release_version
+                    new_url = url
+                    found_release = True
+                    break
+            if not found_release:
+                allow_prerelease = True
+                max_version = release_version
+
+        for version, url in pp_versions[id]:
+            if version.is_prerelease and not allow_prerelease:
+                continue
+            if version > new_version and version < max_version:
+                new_url = url
+                new_version = version
+        lines[index] = line[:start] + new_url + line[end:]
+
+with open('cibuildwheel/windows.py', 'wt') as f:
+    f.writelines(lines)
+
+
+# update Python on macOS
+# Cpython
+# c.f. https://github.com/python/pythondotorg/issues/1352
+response = requests.get('https://www.python.org/api/v2/downloads/release/?version=3&is_published=true')
+response.raise_for_status()
+release_info = response.json()
+cp_versions = {}
+for release in release_info:
+    if not release['is_published']:
+        continue
+    parts = release['name'].split()
+    if parts[0].lower() != 'python':
+        continue
+    assert len(parts) == 2
+    version = packaging.version.parse(parts[1])
+    cp_versions[release['resource_uri']] = [version]
+
+response = requests.get('https://www.python.org/api/v2/downloads/release_file/?os=2')
+response.raise_for_status()
+file_info = response.json()
+
+for file in file_info:
+    key = file['release']
+    if key not in cp_versions.keys():
+        continue
+    cp_versions[key].append(file['url'])
+
+# PyPy is downloaded from https://downloads.python.org/pypy
+response = requests.get('https://downloads.python.org/pypy/versions.json')
+response.raise_for_status()
+pp_realeases = response.json()
+pp_versions = defaultdict(list)
+for pp_realease in pp_realeases:
+    if pp_realease['pypy_version'] == 'nightly':
+        continue
+    version = packaging.version.parse(pp_realease['pypy_version'])
+    python_version = packaging.version.parse(pp_realease['python_version'])
+    python_version = f'{python_version.major}.{python_version.minor}'
+    url = None
+    for file in pp_realease['files']:
+        if f"{file['platform']}-{file['arch']}" == 'darwin-x64':
+            url = file['download_url']
+            break
+    if url:
+        pp_versions[python_version].append((version, url))
+
+# load macos.py
+with open('cibuildwheel/macos.py', 'rt') as f:
+    lines = f.readlines()
+# hugly search pattern, package configuration shall probably done otherwise if we want to do this
+for index, line in enumerate(lines):
+    if 'PythonConfiguration' in line and "identifier='cp3" in line:
+        start = line.index("url='") + 5
+        end = line.index("'", start)
+        current_url = line[start:end]
+        _, current_version_str, installer_kind = current_url.split('/')[-1].split('-')
+        current_version = packaging.version.parse(current_version_str)
+        new_version = current_version
+        new_url = current_url
+        max_version = packaging.version.parse(f'{current_version.major}.{current_version.minor + 1}')
+        allow_prerelease = False
+        if current_version.is_prerelease:
+            release_version = packaging.version.parse(f'{current_version.major}.{current_version.minor}')
+            found_release = False
+            for version in cp_versions.values():
+                if release_version == version[0]:
+                    # find installer
+                    found_url = False
+                    for url in version[1:]:
+                        if url.endswith(installer_kind):
+                            new_url = url
+                            found_url = True
+                            break
+                    if found_url:
+                        new_version = release_version
+                        found_release = True
+                        break
+            if not found_release:
+                allow_prerelease = True
+                max_version = release_version
+
+        for version in cp_versions.values():
+            if version[0].is_prerelease and not allow_prerelease:
+                continue
+            if version[0] > new_version and version[0] < max_version:
+                # check installer kind
+                for url in version[1:]:
+                    if url.endswith(installer_kind):
+                        new_url = url
+                        new_version = version[0]
+        lines[index] = line[:start] + new_url + line[end:]
+    elif 'PythonConfiguration' in line and "identifier='pp" in line:
+        start = line.index("version='") + 9
+        end = line.index("'", start)
+        id = line[start:end]
+        start = line.index("url='") + 5
+        end = line.index("'", start)
+        current_url = line[start:end]
+        _, current_version_str, _ = current_url.split('/')[-1].split('-')
+        current_version = packaging.version.parse(current_version_str)
+        new_version = current_version
+        new_url = current_url
+        max_version = packaging.version.parse(f'{current_version.major}.{current_version.minor + 1}')
+        allow_prerelease = False
+        if current_version.is_prerelease:
+            release_version = packaging.version.parse(f'{current_version.major}.{current_version.minor}')
+            found_release = False
+            for version, url in pp_versions[id]:
+                if release_version == version:
+                    new_version = release_version
+                    new_url = url
+                    found_release = True
+                    break
+            if not found_release:
+                allow_prerelease = True
+                max_version = release_version
+
+        for version, url in pp_versions[id]:
+            if version.is_prerelease and not allow_prerelease:
+                continue
+            if version > new_version and version < max_version:
+                new_url = url
+                new_version = version
+        lines[index] = line[:start] + new_url + line[end:]
+with open('cibuildwheel/macos.py', 'wt') as f:
+    f.writelines(lines)

requirements-dev.txt

@@ -10,3 +10,4 @@ mypy
 pyyaml
 pygithub
 typing-extensions
+packaging>=20.8