zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PythonPackage: install packages with pip (#27798)

Browse Source 
* Use pip to bootstrap pip

* Bootstrap wheel from source

* Update PythonPackage to install using pip

* Update several packages

* Add wheel as base class dep

* Build phase no longer exists

* Add py-poetry package, fix py-flit-core bootstrapping

* Fix isort build

* Clean up many more packages

* Remove unused import

* Fix unit tests

* Don't directly run setup.py

* Typo fix

* Remove unused imports

* Fix issues caught by CI

* Remove custom setup.py file handling

* Use PythonPackage for installing wheels

* Remove custom phases in PythonPackages

* Remove <phase>_args methods

* Remove unused import

* Fix various packages

* Try to test Python packages directly in CI

* Actually run the pipeline

* Fix more packages

* Fix mappings, fix packages

* Fix dep version

* Work around bug in concretizer

* Various concretization fixes

* Fix gitlab yaml, packages

* Fix typo in gitlab yaml

* Skip more packages that fail to concretize

* Fix? jupyter ecosystem concretization issues

* Solve Jupyter concretization issues

* Prevent duplicate entries in PYTHONPATH

* Skip fenics-dolfinx

* Build fewer Python packages

* Fix missing npm dep

* Specify image

* More package fixes

* Add backends for every from-source package

* Fix version arg

* Remove GitLab CI stuff, add py-installer package

* Remove test deps, re-add install_options

* Function declaration syntax fix

* More build fixes

* Update spack create template

* Update PythonPackage documentation

* Fix documentation build

* Fix unit tests

* Remove pip flag added only in newer pip

* flux: add explicit dependency on jsonschema

* Update packages that have been added since this was branched off of develop

* Move Python 2 deprecation to a separate PR

* py-neurolab: add build dep on py-setuptools

* Use wheels for pip/wheel

* Allow use of pre-installed pip for external Python

* pip -> python -m pip

* Use python -m pip for all packages

* Fix py-wrapt

* Add both platlib and purelib to PYTHONPATH

* py-pyyaml: setuptools is needed for all versions

* py-pyyaml: link flags aren't needed

* Appease spack audit packages

* Some build backend is required for all versions, distutils -> setuptools

* Correctly handle different setup.py filename

* Use wheels for py-tomli to avoid circular dep on py-flit-core

* Fix busco installation procedure

* Clarify things in spack create template

* Test other Python build backends

* Undo changes to busco

* Various fixes

* Don't test other backends
This commit is contained in:
Adam J. Stewart
2022-01-14 12:37:57 -06:00
committed by GitHub
parent 0b2507053e
commit 3540f8200a
331 changed files with 1488 additions and 1577 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
lib/spack/spack/build_environment.py
View File

@@ -177,6 +177,7 @@ def clean_environment():
env.unset('OBJC_INCLUDE_PATH')
env.unset('CMAKE_PREFIX_PATH')
env.unset('PYTHONPATH')
# Affects GNU make, can e.g. indirectly inhibit enabling parallel build
env.unset('MAKEFLAGS')
@@ -525,9 +526,10 @@ def _set_variables_for_single_module(pkg, module):
m.cmake = Executable('cmake')
m.ctest = MakeExecutable('ctest', jobs)
# Standard CMake arguments
# Standard build system arguments
m.std_cmake_args = spack.build_systems.cmake.CMakePackage._std_args(pkg)
m.std_meson_args = spack.build_systems.meson.MesonPackage._std_args(pkg)
m.std_pip_args = spack.build_systems.python.PythonPackage._std_args(pkg)
# Put spack compiler paths in module scope.
link_dir = spack.paths.build_env_path

250
lib/spack/spack/build_systems/python.py
View File

@@ -18,65 +18,19 @@
)
from llnl.util.lang import match_predicate
from spack.directives import extends
from spack.directives import depends_on, extends
from spack.package import PackageBase, run_after
class PythonPackage(PackageBase):
"""Specialized class for packages that are built using Python
setup.py files
This class provides the following phases that can be overridden:
* build
* build_py
* build_ext
* build_clib
* build_scripts
* install
* install_lib
* install_headers
* install_scripts
* install_data
These are all standard setup.py commands and can be found by running:
.. code-block:: console
$ python setup.py --help-commands
By default, only the 'build' and 'install' phases are run, but if you
need to run more phases, simply modify your ``phases`` list like so:
.. code-block:: python
phases = ['build_ext', 'install', 'bdist']
Each phase provides a function <phase> that runs:
.. code-block:: console
$ python -s setup.py --no-user-cfg <phase>
Each phase also has a <phase_args> function that can pass arguments to
this call. All of these functions are empty except for the ``install_args``
function, which passes ``--prefix=/path/to/installation/directory``.
If you need to run a phase which is not a standard setup.py command,
you'll need to define a function for it like so:
.. code-block:: python
def configure(self, spec, prefix):
self.setup_py('configure')
"""
"""Specialized class for packages that are built using pip."""
#: Package name, version, and extension on PyPI
pypi = None
maintainers = ['adamjstewart']
# Default phases
phases = ['build', 'install']
phases = ['install']
# To be used in UI queries that require to know which
# build-system class we are using
@@ -86,9 +40,39 @@ def configure(self, spec, prefix):
install_time_test_callbacks = ['test']
extends('python')
depends_on('py-pip', type='build')
# FIXME: technically wheel is only needed when building from source, not when
# installing a downloaded wheel, but I don't want to add wheel as a dep to every
# package manually
depends_on('py-wheel', type='build')
py_namespace = None
@staticmethod
def _std_args(cls):
return [
# Verbose
'-vvv',
# Disable prompting for input
'--no-input',
# Disable the cache
'--no-cache-dir',
# Don't check to see if pip is up-to-date
'--disable-pip-version-check',
# Install packages
'install',
# Don't install package dependencies
'--no-deps',
# Overwrite existing packages
'--ignore-installed',
# Use env vars like PYTHONPATH
'--no-build-isolation',
# Don't warn that prefix.bin is not in PATH
'--no-warn-script-location',
# Ignore the PyPI package index
'--no-index',
]
@property
def homepage(self):
if self.pypi:
@@ -153,163 +137,45 @@ def import_modules(self):
return modules
def setup_file(self):
"""Returns the name of the setup file to use."""
return 'setup.py'
@property
def build_directory(self):
"""The directory containing the ``setup.py`` file."""
"""The root directory of the Python package.
This is usually the directory containing one of the following files:
* ``pyproject.toml``
* ``setup.cfg``
* ``setup.py``
"""
return self.stage.source_path
def python(self, *args, **kwargs):
inspect.getmodule(self).python(*args, **kwargs)
def setup_py(self, *args, **kwargs):
setup = self.setup_file()
with working_dir(self.build_directory):
self.python('-s', setup, '--no-user-cfg', *args, **kwargs)
# The following phases and their descriptions come from:
# $ python setup.py --help-commands
# Standard commands
def build(self, spec, prefix):
"""Build everything needed to install."""
args = self.build_args(spec, prefix)
self.setup_py('build', *args)
def build_args(self, spec, prefix):
"""Arguments to pass to build."""
def install_options(self, spec, prefix):
"""Extra arguments to be supplied to the setup.py install command."""
return []
def build_py(self, spec, prefix):
'''"Build" pure Python modules (copy to build directory).'''
args = self.build_py_args(spec, prefix)
self.setup_py('build_py', *args)
def build_py_args(self, spec, prefix):
"""Arguments to pass to build_py."""
return []
def build_ext(self, spec, prefix):
"""Build C/C++ extensions (compile/link to build directory)."""
args = self.build_ext_args(spec, prefix)
self.setup_py('build_ext', *args)
def build_ext_args(self, spec, prefix):
"""Arguments to pass to build_ext."""
return []
def build_clib(self, spec, prefix):
"""Build C/C++ libraries used by Python extensions."""
args = self.build_clib_args(spec, prefix)
self.setup_py('build_clib', *args)
def build_clib_args(self, spec, prefix):
"""Arguments to pass to build_clib."""
return []
def build_scripts(self, spec, prefix):
'''"Build" scripts (copy and fixup #! line).'''
args = self.build_scripts_args(spec, prefix)
self.setup_py('build_scripts', *args)
def build_scripts_args(self, spec, prefix):
"""Arguments to pass to build_scripts."""
def global_options(self, spec, prefix):
"""Extra global options to be supplied to the setup.py call before the install
or bdist_wheel command."""
return []
def install(self, spec, prefix):
"""Install everything from build directory."""
args = self.install_args(spec, prefix)
self.setup_py('install', *args)
args = PythonPackage._std_args(self) + ['--prefix=' + prefix]
def install_args(self, spec, prefix):
"""Arguments to pass to install."""
args = ['--prefix={0}'.format(prefix)]
for option in self.install_options(spec, prefix):
args.append('--install-option=' + option)
for option in self.global_options(spec, prefix):
args.append('--global-option=' + option)
# This option causes python packages (including setuptools) NOT
# to create eggs or easy-install.pth files. Instead, they
# install naturally into $prefix/pythonX.Y/site-packages.
#
# Eggs add an extra level of indirection to sys.path, slowing
# down large HPC runs. They are also deprecated in favor of
# wheels, which use a normal layout when installed.
#
# Spack manages the package directory on its own by symlinking
# extensions into the site-packages directory, so we don't really
# need the .pth files or egg directories, anyway.
#
# We need to make sure this is only for build dependencies. A package
# such as py-basemap will not build properly with this flag since
# it does not use setuptools to build and those does not recognize
# the --single-version-externally-managed flag
if ('py-setuptools' == spec.name or # this is setuptools, or
'py-setuptools' in spec._dependencies and # it's an immediate dep
'build' in spec._dependencies['py-setuptools'].deptypes):
args += ['--single-version-externally-managed']
if self.stage.archive_file and self.stage.archive_file.endswith('.whl'):
args.append(self.stage.archive_file)
else:
args.append('.')
# Get all relative paths since we set the root to `prefix`
# We query the python with which these will be used for the lib and inc
# directories. This ensures we use `lib`/`lib64` as expected by python.
pkg = spec['python'].package
args += ['--root=%s' % prefix,
'--install-purelib=%s' % pkg.purelib,
'--install-platlib=%s' % pkg.platlib,
'--install-scripts=bin',
'--install-data=',
'--install-headers=%s' % pkg.include,
]
return args
def install_lib(self, spec, prefix):
"""Install all Python modules (extensions and pure Python)."""
args = self.install_lib_args(spec, prefix)
self.setup_py('install_lib', *args)
def install_lib_args(self, spec, prefix):
"""Arguments to pass to install_lib."""
return []
def install_headers(self, spec, prefix):
"""Install C/C++ header files."""
args = self.install_headers_args(spec, prefix)
self.setup_py('install_headers', *args)
def install_headers_args(self, spec, prefix):
"""Arguments to pass to install_headers."""
return []
def install_scripts(self, spec, prefix):
"""Install scripts (Python or otherwise)."""
args = self.install_scripts_args(spec, prefix)
self.setup_py('install_scripts', *args)
def install_scripts_args(self, spec, prefix):
"""Arguments to pass to install_scripts."""
return []
def install_data(self, spec, prefix):
"""Install data files."""
args = self.install_data_args(spec, prefix)
self.setup_py('install_data', *args)
def install_data_args(self, spec, prefix):
"""Arguments to pass to install_data."""
return []
pip = inspect.getmodule(self).pip
with working_dir(self.build_directory):
pip(*args)
# Testing

69
lib/spack/spack/cmd/create.py
View File

@@ -263,19 +263,34 @@ class PythonPackageTemplate(PackageTemplate):
base_class_name = 'PythonPackage'
dependencies = """\
# FIXME: Add dependencies if required. Only add the python dependency
# if you need specific versions. A generic python dependency is
# added implicity by the PythonPackage class.
# FIXME: Only add the python/pip/wheel dependencies if you need specific versions
# or need to change the dependency type. Generic python/pip/wheel dependencies are
# added implicity by the PythonPackage base class.
# depends_on('python@2.X:2.Y,3.Z:', type=('build', 'run'))
# depends_on('py-pip@X.Y:', type='build')
# depends_on('py-wheel@X.Y:', type='build')
# FIXME: Add a build backend, usually defined in pyproject.toml. If no such file
# exists, use setuptools.
# depends_on('py-setuptools', type='build')
# depends_on('py-foo', type=('build', 'run'))"""
# depends_on('py-flit-core', type='build')
# depends_on('py-poetry-core', type='build')
# FIXME: Add additional dependencies if required.
# depends_on('py-foo', type=('build', 'run'))"""
body_def = """\
def build_args(self, spec, prefix):
# FIXME: Add arguments other than --prefix
# FIXME: If not needed delete this function
args = []
return args"""
def global_options(self, spec, prefix):
# FIXME: Add options to pass to setup.py
# FIXME: If not needed, delete this function
options = []
return options
def install_options(self, spec, prefix):
# FIXME: Add options to pass to setup.py install
# FIXME: If not needed, delete this function
options = []
return options"""
def __init__(self, name, url, *args, **kwargs):
# If the user provided `--name py-numpy`, don't rename it py-py-numpy
@@ -298,24 +313,32 @@ def __init__(self, name, url, *args, **kwargs):
# e.g. https://files.pythonhosted.org/packages/c5/63/a48648ebc57711348420670bb074998f79828291f68aebfff1642be212ec/numpy-1.19.4.zip
# e.g. https://files.pythonhosted.org/packages/c5/63/a48648ebc57711348420670bb074998f79828291f68aebfff1642be212ec/numpy-1.19.4.zip#sha256=141ec3a3300ab89c7f2b0775289954d193cc8edb621ea05f99db9cb181530512
# PyPI URLs for wheels are too complicated, ignore them for now
# PyPI URLs for wheels:
# https://pypi.io/packages/py3/a/azureml_core/azureml_core-1.11.0-py3-none-any.whl
# https://pypi.io/packages/py3/d/dotnetcore2/dotnetcore2-2.1.14-py3-none-macosx_10_9_x86_64.whl
# https://pypi.io/packages/py3/d/dotnetcore2/dotnetcore2-2.1.14-py3-none-manylinux1_x86_64.whl
# https://files.pythonhosted.org/packages/cp35.cp36.cp37.cp38.cp39/s/shiboken2/shiboken2-5.15.2-5.15.2-cp35.cp36.cp37.cp38.cp39-abi3-manylinux1_x86_64.whl
# https://files.pythonhosted.org/packages/f4/99/ad2ef1aeeb395ee2319bb981ea08dbbae878d30dd28ebf27e401430ae77a/azureml_core-1.36.0.post2-py3-none-any.whl#sha256=60bcad10b4380d78a8280deb7365de2c2cd66527aacdcb4a173f613876cbe739
match = re.search(
r'(?:pypi|pythonhosted)[^/]+/packages' + '/([^/#]+)' * 4,
url
)
if match:
if len(match.group(2)) == 1:
# Simple PyPI URL
url = '/'.join(match.group(3, 4))
else:
# PyPI URL containing hash
# Project name doesn't necessarily match download name, but it
# usually does, so this is the best we can do
project = parse_name(url)
url = '/'.join([project, match.group(4)])
# PyPI URLs for wheels are too complicated, ignore them for now
# https://www.python.org/dev/peps/pep-0427/#file-name-convention
if not match.group(4).endswith('.whl'):
if len(match.group(2)) == 1:
# Simple PyPI URL
url = '/'.join(match.group(3, 4))
else:
# PyPI URL containing hash
# Project name doesn't necessarily match download name, but it
# usually does, so this is the best we can do
project = parse_name(url)
url = '/'.join([project, match.group(4)])
self.url_line = ' pypi = "{url}"'
self.url_line = ' pypi = "{url}"'
else:
# Add a reminder about spack preferring PyPI URLs
self.url_line = '''
@@ -581,6 +604,9 @@ def __call__(self, stage, url):
if url.endswith('.gem'):
self.build_system = 'ruby'
return
if url.endswith('.whl') or '.whl#' in url:
self.build_system = 'python'
return
# A list of clues that give us an idea of the build system a package
# uses. If the regular expression matches a file contained in the
@@ -596,7 +622,8 @@ def __call__(self, stage, url):
(r'/pom\.xml$', 'maven'),
(r'/SConstruct$', 'scons'),
(r'/waf$', 'waf'),
(r'/setup\.py$', 'python'),
(r'/pyproject.toml', 'python'),
(r'/setup\.(py|cfg)$', 'python'),
(r'/WORKSPACE$', 'bazel'),
(r'/Build\.PL$', 'perlbuild'),
(r'/Makefile\.PL$', 'perlmake'),

8
lib/spack/spack/stage.py
View File

@@ -897,6 +897,10 @@ def get_checksums_for_versions(url_dict, name, **kwargs):
i = 0
errors = []
for url, version in zip(urls, versions):
# Wheels should not be expanded during staging
expand_arg = ''
if url.endswith('.whl') or '.whl#' in url:
expand_arg = ', expand=False'
try:
if fetch_options:
url_or_fs = fs.URLFetchStrategy(
@@ -931,8 +935,8 @@ def get_checksums_for_versions(url_dict, name, **kwargs):
# Generate the version directives to put in a package.py
version_lines = "\n".join([
" version('{0}', {1}sha256='{2}')".format(
v, ' ' * (max_len - len(str(v))), h) for v, h in version_hashes
" version('{0}', {1}sha256='{2}'{3})".format(
v, ' ' * (max_len - len(str(v))), h, expand_arg) for v, h in version_hashes
])
num_hash = len(version_hashes)

2
lib/spack/spack/test/cmd/create.py
View File

@@ -63,7 +63,7 @@ def parser():
r'def configure_args(self']),
(['-t', 'python', 'test-python'], 'py-test-python',
[r'PyTestPython(PythonPackage)', r"depends_on('py-",
r'def build_args(self']),
r'def global_options(self', r'def install_options(self']),
(['-t', 'qmake', 'test-qmake'], 'test-qmake',
[r'TestQmake(QMakePackage)', r'def qmake_args(self']),
(['-t', 'r', 'test-r'], 'r-test-r',