From 63dea27461d9d7e9c5c95d9ce69c003efb38e0ef Mon Sep 17 00:00:00 2001 From: Jorge Capona Date: Mon, 23 Oct 2023 12:40:36 -0300 Subject: [PATCH] Migrate to use PyPI's JSON API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PyPI’s XML-RPC API is deprecated, causing errors when downloading packages --- setup.py | 1 + stdeb/downloader.py | 50 +++++----------------- stdeb/transport.py | 100 -------------------------------------------- 3 files changed, 12 insertions(+), 139 deletions(-) delete mode 100644 stdeb/transport.py diff --git a/setup.py b/setup.py index d5308dd..b119398 100644 --- a/setup.py +++ b/setup.py @@ -32,4 +32,5 @@ 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', ], + requires=['setuptools', 'requests'], ) diff --git a/stdeb/downloader.py b/stdeb/downloader.py index 63f5aee..a188373 100644 --- a/stdeb/downloader.py +++ b/stdeb/downloader.py @@ -1,18 +1,10 @@ from __future__ import print_function import os -try: - # Python 2.x - import xmlrpclib -except ImportError: - # Python 3.x - import xmlrpc.client as xmlrpclib from functools import partial import requests import hashlib import warnings import stdeb -from stdeb.transport import RequestsTransport -import time myprint = print @@ -22,11 +14,10 @@ def find_tar_gz(package_name, pypi_url='https://pypi.org', verbose=0, release=None): - transport = RequestsTransport() - transport.user_agent = USER_AGENT - if pypi_url.startswith('https://'): - transport.use_https = True - pypi = xmlrpclib.ServerProxy(pypi_url, transport=transport) + response = requests.get(f"{pypi_url}/pypi/{package_name}/json") + if response.status_code != 200: + raise ValueError('PyPI returned request with status code %d' % response.status_code) + pypi = response.json() download_url = None expected_md5_digest = None @@ -35,8 +26,8 @@ def find_tar_gz(package_name, pypi_url='https://pypi.org', myprint('querying PyPI (%s) for package name "%s"' % (pypi_url, package_name)) - show_hidden = True - all_releases = _call(pypi.package_releases, package_name, show_hidden) + all_releases = pypi.get('releases') + if release is not None: # A specific release is requested. if verbose >= 2: @@ -49,19 +40,17 @@ def find_tar_gz(package_name, pypi_url='https://pypi.org', 'releases %r' % (release, all_releases)) version = release else: - default_releases = _call(pypi.package_releases, package_name) - if len(default_releases) != 1: + default_release = pypi.get('info', {}).get('version') + if default_release is None: raise RuntimeError('Expected one and only one release. ' 'Non-hidden: %r. All: %r' % - (default_releases, all_releases)) - default_release = default_releases[0] + (default_release, all_releases.keys())) if verbose >= 2: - myprint( - 'found default release: %s' % (', '.join(default_releases),)) + myprint('found default release: %s' % default_release) version = default_release - urls = _call(pypi.release_urls, package_name, version) + urls = all_releases.get(version) for url in urls: if url['packagetype'] == 'sdist': assert url['python_version'] == 'source', \ @@ -72,28 +61,11 @@ def find_tar_gz(package_name, pypi_url='https://pypi.org', expected_md5_digest = url['md5_digest'] break - if download_url is None: - # PyPI doesn't have package. Is download URL provided? - result = _call(pypi.release_data, package_name, version) - if result['download_url'] != 'UNKNOWN': - download_url = result['download_url'] - # no download URL provided, see if PyPI itself has download - urls = _call(pypi.release_urls, result['name'], result['version']) if download_url is None: raise ValueError('no package "%s" was found' % package_name) return download_url, expected_md5_digest -def _call(callable_, *args, **kwargs): - try: - return callable_(*args, **kwargs) - except xmlrpclib.Fault as e: - if not e.faultString.startswith('HTTPTooManyRequests'): - raise - time.sleep(1) # try again after rate limit - return callable_(*args, **kwargs) - - def md5sum(filename): # from # http://stackoverflow.com/questions/7829499/using-hashlib-to-compute-md5-digest-of-a-file-in-python-3 diff --git a/stdeb/transport.py b/stdeb/transport.py deleted file mode 100644 index 638b806..0000000 --- a/stdeb/transport.py +++ /dev/null @@ -1,100 +0,0 @@ -# -*- coding: utf-8 -*- -""" -A replacement transport for Python xmlrpc library. - -Usage: - - >>> import xmlrpclib - >>> from transport import RequestsTransport - >>> s = xmlrpclib.ServerProxy( - >>> 'http://yoursite.com/xmlrpc', transport=RequestsTransport()) - >>> s.demo.sayHello() - Hello! -""" -try: - import xmlrpc.client as xmlrpc -except ImportError: - import xmlrpclib as xmlrpc - -import requests -import requests.utils - -import sys -from distutils.version import StrictVersion -import warnings - - -class RequestsTransport(xmlrpc.Transport): - """ - Drop in Transport for xmlrpclib that uses Requests instead of httplib - """ - # change our user agent to reflect Requests - user_agent = "Python XMLRPC with Requests (python-requests.org)" - - # override this if you'd like to https - use_https = False - - def request(self, host, handler, request_body, verbose): - """ - Make an xmlrpc request. - """ - headers = {'User-Agent': self.user_agent, - 'Content-Type': 'text/xml', - } - url = self._build_url(host, handler) - kwargs = {} - if StrictVersion(requests.__version__) >= StrictVersion('0.8.8'): - kwargs['verify'] = True - else: - if self.use_https: - warnings.warn( - 'using https transport but no certificate ' - 'verification. (Hint: upgrade requests package.)') - try: - resp = requests.post(url, data=request_body, headers=headers, - **kwargs) - except ValueError: - raise - except Exception: - raise # something went wrong - else: - try: - resp.raise_for_status() - except requests.RequestException as e: - raise xmlrpc.ProtocolError( - url, resp.status_code, str(e), resp.headers) - else: - return self.parse_response(resp) - - def parse_response(self, resp): - """ - Parse the xmlrpc response. - """ - p, u = self.getparser() - - if hasattr(resp, 'text'): - # modern requests will do this for us - text = resp.text # this is unicode(py2)/str(py3) - else: - - encoding = requests.utils.get_encoding_from_headers(resp.headers) - if encoding is None: - encoding = 'utf-8' # FIXME: what to do here? - - if sys.version_info[0] == 2: - text = unicode( # noqa: F821 - resp.content, encoding, errors='replace') - else: - assert sys.version_info[0] == 3 - text = str(resp.content, encoding, errors='replace') - p.feed(text) - p.close() - return u.close() - - def _build_url(self, host, handler): - """ - Build a url for our request based on the host, handler and use_http - property - """ - scheme = 'https' if self.use_https else 'http' - return '%s://%s/%s' % (scheme, host, handler)