zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PythonPackage: add pypi attribute to infer homepage/url/list_url (#17587)

Browse Source
This commit is contained in:
Adam J. Stewart 2020-12-29 02:03:08 -06:00 committed by GitHub
parent 76d23d9ee4
commit 05f8e08067
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 122 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
lib/spack/docs/build_systems/pythonpackage.rst
View File

@ -134,9 +134,9 @@ The zip file will not contain a ``setup.py``, but it will contain a
``METADATA`` file which contains all the information you need to
write a ``package.py`` build recipe.
^^^^^^^^^^^^^^^^^^^^^^^
Finding Python packages
^^^^^^^^^^^^^^^^^^^^^^^
^^^^
PyPI
^^^^
The vast majority of Python packages are hosted on PyPI - The Python
Package Index. ``pip`` only supports packages hosted on PyPI, making
@ -148,6 +148,26 @@ if a newer version is available. The download page is usually at::
https://pypi.org/project/<package-name>
Since PyPI is so common, the ``PythonPackage`` base class has a
``pypi`` attribute that can be set. Once set, ``pypi`` will be used
to define the ``homepage``, ``url``, and ``list_url``. For example,
the following:
.. code-block:: python
homepage = 'https://pypi.org/project/setuptools/'
url = 'https://pypi.org/packages/source/s/setuptools/setuptools-49.2.0.zip'
list_url = 'https://pypi.org/simple/setuptools/'
is equivalent to:
.. code-block:: python
pypi = 'setuptools/setuptools-49.2.0.zip'
^^^^^^^^^^^
Description
^^^^^^^^^^^
@ -184,50 +204,11 @@ also get the homepage on the command-line by running:
URL
^^^
You may have noticed that Spack allows you to add multiple versions of
the same package without adding multiple versions of the download URL.
It does this by guessing what the version string in the URL is and
replacing this with the requested version. Obviously, if Spack cannot
guess the version correctly, or if non-version-related things change
in the URL, Spack cannot substitute the version properly.
Once upon a time, PyPI offered nice, simple download URLs like::
https://pypi.python.org/packages/source/n/numpy/numpy-1.13.1.zip
As you can see, the version is 1.13.1. It probably isn't hard to guess
what URL to use to download version 1.12.0, and Spack was perfectly
capable of performing this calculation.
However, PyPI switched to a new download URL format::
https://pypi.python.org/packages/c0/3a/40967d9f5675fbb097ffec170f59c2ba19fc96373e73ad47c2cae9a30aed/numpy-1.13.1.zip#md5=2c3c0f4edf720c3a7b525dacc825b9ae
and more recently::
https://files.pythonhosted.org/packages/b0/2b/497c2bb7c660b2606d4a96e2035e92554429e139c6c71cdff67af66b58d2/numpy-1.14.3.zip
As you can imagine, it is impossible for Spack to guess what URL to
use to download version 1.12.0 given this URL. There is a solution,
however. PyPI offers a new hidden interface for downloading
Python packages that does not include a hash in the URL::
https://pypi.io/packages/source/n/numpy/numpy-1.13.1.zip
This URL redirects to the https://files.pythonhosted.org URL. The general
syntax for this https://pypi.io URL is::
https://pypi.io/packages/<type>/<first-letter-of-name>/<name>/<name>-<version>.<extension>
Please use the https://pypi.io URL instead of the https://pypi.python.org
URL. If both ``.tar.gz`` and ``.zip`` versions are available, ``.tar.gz``
is preferred. If some releases offer both ``.tar.gz`` and ``.zip`` versions,
but some only offer ``.zip`` versions, use ``.zip``.
If ``pypi`` is set as mentioned above, ``url`` and ``list_url`` will
be automatically set for you. If both ``.tar.gz`` and ``.zip`` versions
are available, ``.tar.gz`` is preferred. If some releases offer both
``.tar.gz`` and ``.zip`` versions, but some only offer ``.zip`` versions,
use ``.zip``.
Some Python packages are closed-source and do not ship ``.tar.gz`` or ``.zip``
files on either PyPI or GitHub. If this is the case, you can still download
@ -237,10 +218,9 @@ and can be downloaded from::
https://pypi.io/packages/py3/a/azureml_sdk/azureml_sdk-1.11.0-py3-none-any.whl
Note that instead of ``<type>`` being ``source``, it is now ``py3`` since this
wheel will work for any generic version of Python 3. You may see Python-specific
or OS-specific URLs. Note that when you add a ``.whl`` URL, you should add
``expand=False`` to ensure that Spack doesn't try to extract the wheel:
You may see Python-specific or OS-specific URLs. Note that when you add a
``.whl`` URL, you should add ``expand=False`` to ensure that Spack doesn't
try to extract the wheel:
.. code-block:: python

23
lib/spack/spack/build_systems/python.py
View File

@ -72,6 +72,9 @@ class PythonPackage(PackageBase):
def configure(self, spec, prefix):
self.setup_py('configure')
"""
#: Package name, version, and extension on PyPI
pypi = None
# Default phases
phases = ['build', 'install']
@ -88,6 +91,26 @@ def configure(self, spec, prefix):
py_namespace = None
@property
def homepage(self):
if self.pypi:
name = self.pypi.split('/')[0]
return 'https://pypi.org/project/' + name + '/'
@property
def url(self):
if self.pypi:
return (
'https://files.pythonhosted.org/packages/source/'
+ self.pypi[0] + '/' + self.pypi
)
@property
def list_url(self):
if self.pypi:
name = self.pypi.split('/')[0]
return 'https://pypi.org/simple/' + name + '/'
@property
def import_modules(self):
"""Names of modules that the Python package provides.

42
lib/spack/spack/cmd/create.py
View File

@ -117,7 +117,7 @@ def install(self, spec, prefix):
make()
make('install')"""
url_line = """ url = \"{url}\""""
url_line = ' url = "{url}"'
def __init__(self, name, url, versions):
super(PackageTemplate, self).__init__(name, versions)
@ -270,14 +270,47 @@ def build_args(self, spec, prefix):
args = []
return args"""
def __init__(self, name, *args, **kwargs):
def __init__(self, name, url, *args, **kwargs):
# If the user provided `--name py-numpy`, don't rename it py-py-numpy
if not name.startswith('py-'):
# Make it more obvious that we are renaming the package
tty.msg("Changing package name from {0} to py-{0}".format(name))
name = 'py-{0}'.format(name)
super(PythonPackageTemplate, self).__init__(name, *args, **kwargs)
# Simple PyPI URLs:
# https://<hostname>/packages/<type>/<first character of project>/<project>/<download file>
# e.g. https://pypi.io/packages/source/n/numpy/numpy-1.19.4.zip
# e.g. https://www.pypi.io/packages/source/n/numpy/numpy-1.19.4.zip
# e.g. https://pypi.org/packages/source/n/numpy/numpy-1.19.4.zip
# e.g. https://pypi.python.org/packages/source/n/numpy/numpy-1.19.4.zip
# e.g. https://files.pythonhosted.org/packages/source/n/numpy/numpy-1.19.4.zip
# PyPI URLs containing hash:
# https://<hostname>/packages/<two character hash>/<two character hash>/<longer hash>/<download file>
# e.g. https://pypi.io/packages/c5/63/a48648ebc57711348420670bb074998f79828291f68aebfff1642be212ec/numpy-1.19.4.zip
# e.g. https://files.pythonhosted.org/packages/c5/63/a48648ebc57711348420670bb074998f79828291f68aebfff1642be212ec/numpy-1.19.4.zip
# e.g. https://files.pythonhosted.org/packages/c5/63/a48648ebc57711348420670bb074998f79828291f68aebfff1642be212ec/numpy-1.19.4.zip#sha256=141ec3a3300ab89c7f2b0775289954d193cc8edb621ea05f99db9cb181530512
# PyPI URLs for wheels are too complicated, ignore them for now
match = re.search(
r'(?:pypi|pythonhosted)[^/]+/packages' + '/([^/#]+)' * 4,
url
)
if match:
if len(match.group(2)) == 1:
# Simple PyPI URL
url = '/'.join(match.group(3, 4))
else:
# PyPI URL containing hash
# Project name doesn't necessarily match download name, but it
# usually does, so this is the best we can do
project = parse_name(url)
url = '/'.join([project, match.group(4)])
self.url_line = ' pypi = "{url}"'
super(PythonPackageTemplate, self).__init__(name, url, *args, **kwargs)
class RPackageTemplate(PackageTemplate):
@ -545,7 +578,8 @@ def __call__(self, stage, url):
]
# Peek inside the compressed file.
if stage.archive_file.endswith('.zip'):
if (stage.archive_file.endswith('.zip') or
'.zip#' in stage.archive_file):
try:
unzip = which('unzip')
output = unzip('-lq', stage.archive_file, output=str)

16
lib/spack/spack/package.py
View File

@ -646,6 +646,15 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)):
#: index of patches by sha256 sum, built lazily
_patches_by_hash = None
#: Package homepage where users can find more information about the package
homepage = None
#: Default list URL (place to find available versions)
list_url = None
#: Link depth to which list_url should be searched for new versions
list_depth = 0
#: List of strings which contains GitHub usernames of package maintainers.
#: Do not include @ here in order not to unnecessarily ping the users.
maintainers = [] # type: List[str]
@ -683,13 +692,6 @@ def __init__(self, spec):
msg += " [package '{0.name}' defines both]"
raise ValueError(msg.format(self))
# Set a default list URL (place to find available versions)
if not hasattr(self, 'list_url'):
self.list_url = None
if not hasattr(self, 'list_depth'):
self.list_depth = 0
# init internal variables
self._stage = None
self._fetcher = None

14
lib/spack/spack/url.py
View File

@ -56,8 +56,12 @@ def find_list_urls(url):
GitLab https://gitlab.\*/<repo>/<name>/tags
BitBucket https://bitbucket.org/<repo>/<name>/downloads/?tab=tags
CRAN https://\*.r-project.org/src/contrib/Archive/<name>
PyPI https://pypi.org/simple/<name>/
========= =======================================================
Note: this function is called by `spack versions`, `spack checksum`,
and `spack create`, but not by `spack fetch` or `spack install`.
Parameters:
url (str): The download URL for the package
@ -91,6 +95,16 @@ def find_list_urls(url):
# e.g. https://cloud.r-project.org/src/contrib/rgl_0.98.1.tar.gz
(r'(.*\.r-project\.org/src/contrib)/([^_]+)',
lambda m: m.group(1) + '/Archive/' + m.group(2)),
# PyPI
# e.g. https://pypi.io/packages/source/n/numpy/numpy-1.19.4.zip
# e.g. https://www.pypi.io/packages/source/n/numpy/numpy-1.19.4.zip
# e.g. https://pypi.org/packages/source/n/numpy/numpy-1.19.4.zip
# e.g. https://pypi.python.org/packages/source/n/numpy/numpy-1.19.4.zip
# e.g. https://files.pythonhosted.org/packages/source/n/numpy/numpy-1.19.4.zip
# e.g. https://pypi.io/packages/py2.py3/o/opencensus-context/opencensus_context-0.1.1-py2.py3-none-any.whl
(r'(?:pypi|pythonhosted)[^/]+/packages/[^/]+/./([^/]+)',
lambda m: 'https://pypi.org/simple/' + m.group(1) + '/'),
]
list_urls = set([os.path.dirname(url)])

5
lib/spack/spack/util/web.py
View File

@ -554,7 +554,10 @@ def find_versions_of_archive(
# .sha256
# .sig
# However, SourceForge downloads still need to end in '/download'.
url_regex += r'(\/download)?$'
url_regex += r'(\/download)?'
# PyPI adds #sha256=... to the end of the URL
url_regex += '(#sha256=.*)?'
url_regex += '$'
regexes.append(url_regex)

2
var/spack/repos/builtin/packages/py-matplotlib/package.py
View File

@ -11,7 +11,7 @@ class PyMatplotlib(PythonPackage):
and interactive visualizations in Python."""
homepage = "https://matplotlib.org/"
url = "https://pypi.io/packages/source/m/matplotlib/matplotlib-3.3.2.tar.gz"
pypi = "matplotlib/matplotlib-3.3.2.tar.gz"
maintainers = ['adamjstewart']
import_modules = [

2
var/spack/repos/builtin/packages/py-numpy/package.py
View File

@ -16,7 +16,7 @@ class PyNumpy(PythonPackage):
number capabilities"""
homepage = "https://numpy.org/"
url = "https://pypi.io/packages/source/n/numpy/numpy-1.19.4.zip"
pypi = "numpy/numpy-1.19.4.zip"
git = "https://github.com/numpy/numpy.git"
maintainers = ['adamjstewart']

2
var/spack/repos/builtin/packages/py-scipy/package.py
View File

@ -12,7 +12,7 @@ class PyScipy(PythonPackage):
as routines for numerical integration and optimization."""
homepage = "https://www.scipy.org/"
url = "https://pypi.io/packages/source/s/scipy/scipy-1.5.4.tar.gz"
pypi = "scipy/scipy-1.5.4.tar.gz"
git = "https://github.com/scipy/scipy.git"
maintainers = ['adamjstewart']