diff --git a/Vagrantfile.debian-8-jessie.rb b/Vagrantfile.debian-10-bookworm.rb similarity index 100% rename from Vagrantfile.debian-8-jessie.rb rename to Vagrantfile.debian-10-bookworm.rb diff --git a/stdeb/downloader.py b/stdeb/downloader.py index 63f5aee8..032e0f1d 100644 --- a/stdeb/downloader.py +++ b/stdeb/downloader.py @@ -1,18 +1,11 @@ from __future__ import print_function import os -try: - # Python 2.x - import xmlrpclib -except ImportError: - # Python 3.x - import xmlrpc.client as xmlrpclib from functools import partial import requests import hashlib import warnings import stdeb -from stdeb.transport import RequestsTransport -import time +from stdeb.pypi_simple import PyPIClient myprint = print @@ -22,11 +15,7 @@ def find_tar_gz(package_name, pypi_url='https://pypi.org', verbose=0, release=None): - transport = RequestsTransport() - transport.user_agent = USER_AGENT - if pypi_url.startswith('https://'): - transport.use_https = True - pypi = xmlrpclib.ServerProxy(pypi_url, transport=transport) + pypi_client = PyPIClient(pypi_url, USER_AGENT) download_url = None expected_md5_digest = None @@ -35,8 +24,7 @@ def find_tar_gz(package_name, pypi_url='https://pypi.org', myprint('querying PyPI (%s) for package name "%s"' % (pypi_url, package_name)) - show_hidden = True - all_releases = _call(pypi.package_releases, package_name, show_hidden) + all_releases = pypi_client.release_versions(package_name) if release is not None: # A specific release is requested. if verbose >= 2: @@ -49,51 +37,20 @@ def find_tar_gz(package_name, pypi_url='https://pypi.org', 'releases %r' % (release, all_releases)) version = release else: - default_releases = _call(pypi.package_releases, package_name) - if len(default_releases) != 1: - raise RuntimeError('Expected one and only one release. ' - 'Non-hidden: %r. All: %r' % - (default_releases, all_releases)) - default_release = default_releases[0] + default_release = pypi_client.release_version(package_name) if verbose >= 2: myprint( - 'found default release: %s' % (', '.join(default_releases),)) + 'found default release: %s' % (', '.join(default_release),)) version = default_release - urls = _call(pypi.release_urls, package_name, version) - for url in urls: - if url['packagetype'] == 'sdist': - assert url['python_version'] == 'source', \ - 'how can an sdist not be a source?' - if url['url'].endswith(('.tar.gz', '.zip')): - download_url = url['url'] - if 'md5_digest' in url: - expected_md5_digest = url['md5_digest'] - break + download_url, expected_md5_digest = pypi_client.download_url(package_name, version) - if download_url is None: - # PyPI doesn't have package. Is download URL provided? - result = _call(pypi.release_data, package_name, version) - if result['download_url'] != 'UNKNOWN': - download_url = result['download_url'] - # no download URL provided, see if PyPI itself has download - urls = _call(pypi.release_urls, result['name'], result['version']) if download_url is None: raise ValueError('no package "%s" was found' % package_name) return download_url, expected_md5_digest -def _call(callable_, *args, **kwargs): - try: - return callable_(*args, **kwargs) - except xmlrpclib.Fault as e: - if not e.faultString.startswith('HTTPTooManyRequests'): - raise - time.sleep(1) # try again after rate limit - return callable_(*args, **kwargs) - - def md5sum(filename): # from # http://stackoverflow.com/questions/7829499/using-hashlib-to-compute-md5-digest-of-a-file-in-python-3 diff --git a/stdeb/pypi_simple.py b/stdeb/pypi_simple.py new file mode 100644 index 00000000..fc952b3e --- /dev/null +++ b/stdeb/pypi_simple.py @@ -0,0 +1,45 @@ +import re +import requests +from stdeb.downloader import USER_AGENT + + +def normalize_package_name(package_name): + return re.sub(r"[-_.]+", "-", package_name).lower() + + +class PyPIClient(object): + def __init__(self, pypi_url="https://pypi.org", user_agent=USER_AGENT): + self.pypi_url = pypi_url + self._http_session = requests.Session() + self._http_session.headers["User-Agent"] = user_agent + + def release_version(self, package_name): + package_name = normalize_package_name(package_name) + response = self._http_session.get("%s/pypi/%s/json" % (self.pypi_url, package_name)) + data = response.json() + return data["info"]["version"] + + def release_versions(self, package_name): + package_name = normalize_package_name(package_name) + response = self._http_session.get( + "%s/simple/%s" % (self.pypi_url, package_name), + headers={"Accept": "application/vnd.pypi.simple.latest+json"}) + data = response.json() + return data["versions"] + + def download_url(self, package_name, version): + download_url = None + md5_digest = None + package_name = normalize_package_name(package_name) + response = self._http_session.get( + "%s/pypi/%s/%s/json" % (self.pypi_url, package_name, version)) + data = response.json() + for url in data["urls"]: + if url["packagetype"] == "sdist" and \ + url["python_version"] == "source" and \ + url["url"].endswith((".tar.gz", ".zip")): + download_url = url["url"] + md5_digest = url["digests"]["md5"] + break + + return download_url, md5_digest